Postgresql ha (#89)

* wip * updated dev launch config * handle storage backend reconnect events
3 years ago · 482e01cce2
parent a5d0395417
commit 482e01cce2
12 changed files with 253 additions and 44 deletions
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@ -13,13 +13,13 @@
      "env": {
        "WG_ADMIN_PASSWORD": "example",
        "WG_WIREGUARD_PRIVATE_KEY": "4DRYOeSSeZyWRrLw357Pg9sv/RppMGwveTwz7sxM4mo=",
+        "WG_STORAGE": "sqlite3:///tmp/wg-access-server.sqlite3"
      },
      "args": [
        "serve",
        "--config=config.yaml",
        "--no-wireguard-enabled",
-        "--no-dns-enabled",
-        "--port=9001"
+        "--no-dns-enabled"
      ]
    }
  ]
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

+## [next]
+
+- High availability (HA) is now supported using a Postgres storage backend
+  - TODO: link to docs
+- The file:// storage backend has been removed in favour of sqlite:// for local filesystem persistence
+- The wireguard service can now be disabled via the config file. Helpful for developing on Mac and Windows until support for Mac/Windows networking is added.
+
 ## [v0.3.0]

 ### Added
--- a/go.mod
+++ b/go.mod
@ -18,19 +18,18 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.0
 	github.com/improbable-eng/grpc-web v0.13.0
 	github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07 // indirect
-	github.com/jinzhu/gorm v1.9.14
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/lib/pq v1.7.0 // indirect
+	github.com/jinzhu/gorm v1.9.16
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
 	github.com/miekg/dns v1.1.30
 	github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
+	github.com/place1/pg-events v0.2.0
 	github.com/place1/wg-embed v0.4.0
-	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
+	github.com/pquerna/cachecontrol v0.0.0-20200921180117-858c6e7e6b7e // indirect
 	github.com/rs/cors v1.7.0 // indirect
-	github.com/sirupsen/logrus v1.6.0
-	github.com/stretchr/testify v1.4.0
+	github.com/sirupsen/logrus v1.7.0
+	github.com/stretchr/testify v1.6.1
 	github.com/tg123/go-htpasswd v1.0.0
 	github.com/vishvananda/netlink v1.1.0
 	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
@ -42,7 +41,6 @@ require (
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/Knetic/govaluate.v2 v2.3.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v2 v2.3.0
 	gotest.tools v2.2.0+incompatible // indirect
--- a/go.sum
+++ b/go.sum
@ -86,8 +86,8 @@ github.com/improbable-eng/grpc-web v0.13.0 h1:7XqtaBWaOCH0cVGKHyvhtcuo6fgW32Y10y
 github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs=
 github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07 h1:rw3IAne6CDuVFlZbPOkA7bhxlqawFh7RJJ+CejfMaxE=
 github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg=
-github.com/jinzhu/gorm v1.9.14 h1:Kg3ShyTPcM6nzVo148fRrcMO6MNKuqtOUwnzqMgVniM=
-github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
+github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o=
+github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
@ -105,15 +105,15 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY=
-github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
+github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
@ -138,12 +138,14 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/place1/pg-events v0.2.0 h1:7v8byv7GO8Dc2ufdgRgma5RAraC5sOe6q+jnOhN+dk0=
+github.com/place1/pg-events v0.2.0/go.mod h1:IwHKE93V/uyZWui7MY1iaEYbz8MdqJnGbYOSOCICKbo=
 github.com/place1/wg-embed v0.4.0 h1:rToHj4+TuI2ruv2mz3Y16vvisv280BuzdojsGGNQ/pM=
 github.com/place1/wg-embed v0.4.0/go.mod h1:i09dm8AEkurC4oATFxjvyH0+e1pdmtZoNk2FfPupROI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
-github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/pquerna/cachecontrol v0.0.0-20200921180117-858c6e7e6b7e h1:BLqxdwZ6j771IpSCRx7s/GJjXHUE00Hmu7/YegCGdzA=
+github.com/pquerna/cachecontrol v0.0.0-20200921180117-858c6e7e6b7e/go.mod h1:hoLfEwdY11HjRfKFH6KqnPsfxlo3BP6bJehpDv8t6sQ=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
@ -152,6 +154,8 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
@ -163,6 +167,8 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tg123/go-htpasswd v1.0.0 h1:Ze/pZsz73JiCwXIyJBPvNs75asKBgfodCf8iTEkgkXs=
 github.com/tg123/go-htpasswd v1.0.0/go.mod h1:eQTgl67UrNKQvEPKrDLGBssjVwYQClFZjALVLhIv8C0=
 github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
@ -231,12 +237,14 @@ golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -296,16 +304,19 @@ gopkg.in/Knetic/govaluate.v2 v2.3.0/go.mod h1:NW0gr10J8s7aNghEg6uhdxiEaBvc0+8VgJ
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
 gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w=
 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
--- a/internal/devices/devices.go
+++ b/internal/devices/devices.go
@ -25,15 +25,28 @@ func New(wg wgembed.WireGuardInterface, s storage.Storage, cidr string) *DeviceM
 }

 func (d *DeviceManager) StartSync(disableMetadataCollection bool) error {
-	// sync devices from storage once
-	devices, err := d.ListDevices("")
-	if err != nil {
-		return errors.Wrap(err, "failed to list devices")
-	}
-	for _, device := range devices {
+	// Start listening to the device add/remove events
+	d.storage.OnAdd(func(device *storage.Device) {
 		if err := d.wg.AddPeer(device.PublicKey, device.Address); err != nil {
-			logrus.Warn(errors.Wrapf(err, "failed to sync device '%s' (ignoring)", device.Name))
+			logrus.Error(errors.Wrap(err, "failed to add wireguard peer"))
 		}
+	})
+
+	d.storage.OnDelete(func(device *storage.Device) {
+		if err := d.wg.RemovePeer(device.PublicKey); err != nil {
+			logrus.Error(errors.Wrap(err, "failed to remove wireguard peer"))
+		}
+	})
+
+	d.storage.OnReconnect(func() {
+		if err := d.sync(); err != nil {
+			logrus.Error(errors.Wrap(err, "device sync after storage backend reconnect event failed"))
+		}
+	})
+
+	// Do an initial sync of existing devices
+	if err := d.sync(); err != nil {
+		return errors.Wrap(err, "initial device sync from storage failed")
 	}

 	// start the metrics loop
@ -69,10 +82,6 @@ func (d *DeviceManager) AddDevice(identity *authsession.Identity, name string, p
 		return nil, errors.Wrap(err, "failed to save the new device")
 	}

-	if err := d.wg.AddPeer(publicKey, clientAddr); err != nil {
-		return nil, errors.Wrap(err, "unable to provision peer")
-	}
-
 	return device, nil
 }

@ -80,6 +89,36 @@ func (d *DeviceManager) SaveDevice(device *storage.Device) error {
 	return d.storage.Save(device)
 }

+func (d *DeviceManager) sync() error {
+	devices, err := d.ListAllDevices()
+	if err != nil {
+		return errors.Wrap(err, "failed to list devices")
+	}
+
+	peers, err := d.wg.ListPeers()
+	if err != nil {
+		return errors.Wrap(err, "failed to list peers")
+	}
+
+	// Remove any peers for devices that are no longer in storage
+	for _, peer := range peers {
+		if !deviceListContains(devices, peer.PublicKey.String()) {
+			if err := d.wg.RemovePeer(peer.PublicKey.String()); err != nil {
+				logrus.Error(errors.Wrapf(err, "failed to remove peer during sync: %s", peer.PublicKey.String()))
+			}
+		}
+	}
+
+	// Add peers for all devices in storage
+	for _, device := range devices {
+		if err := d.wg.AddPeer(device.PublicKey, device.Address); err != nil {
+			logrus.Warn(errors.Wrapf(err, "failed to add device during sync: %s", device.Name))
+		}
+	}
+
+	return nil
+}
+
 func (d *DeviceManager) ListAllDevices() ([]*storage.Device, error) {
 	return d.storage.List("")
 }
@ -93,12 +132,11 @@ func (d *DeviceManager) DeleteDevice(user string, name string) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to retrieve device")
 	}
+
 	if err := d.storage.Delete(device); err != nil {
 		return err
 	}
-	if err := d.wg.RemovePeer(device.PublicKey); err != nil {
-		return errors.Wrap(err, "device was removed from storage but failed to be removed from the wireguard interface")
-	}
+
 	return nil
 }

@ -169,3 +207,12 @@ func nextIP(ip net.IP) net.IP {
 	}
 	return next
 }
+
+func deviceListContains(devices []*storage.Device, publicKey string) bool {
+	for _, device := range devices {
+		if device.PublicKey == publicKey {
+			return true
+		}
+	}
+	return false
+}
--- a/internal/storage/contracts.go
+++ b/internal/storage/contracts.go
@ -10,6 +10,7 @@ import (
 )

 type Storage interface {
+	Watcher
 	Save(device *Device) error
 	List(owner string) ([]*Device, error)
 	Get(owner string, name string) (*Device, error)
@ -18,15 +19,23 @@ type Storage interface {
 	Open() error
 }

+type Watcher interface {
+	OnAdd(cb Callback)
+	OnDelete(cb Callback)
+	OnReconnect(func())
+}
+
+type Callback func(device *Device)
+
 type Device struct {
 	Owner         string    `json:"owner" gorm:"type:varchar(100);unique_index:key;primary_key"`
-	OwnerName     string    `json:"ownerName"`
-	OwnerEmail    string    `json:"ownerEmail"`
-	OwnerProvider string    `json:"ownerProvider"`
+	OwnerName     string    `json:"owner_name"`
+	OwnerEmail    string    `json:"owner_email"`
+	OwnerProvider string    `json:"owner_provider"`
 	Name          string    `json:"name" gorm:"type:varchar(100);unique_index:key;primary_key"`
-	PublicKey     string    `json:"publicKey"`
+	PublicKey     string    `json:"public_key"`
 	Address       string    `json:"address"`
-	CreatedAt     time.Time `json:"createdAt" gorm:"column:created_at"`
+	CreatedAt     time.Time `json:"created_at" gorm:"column:created_at"`

 	/**
 	 * Metadata fields below.
@ -35,9 +44,9 @@ type Device struct {
 	 */

 	// metadata about the device during the current session
-	LastHandshakeTime *time.Time `json:"lastHandshakeTime"`
-	ReceiveBytes      int64      `json:"receivedBytes"`
-	TransmitBytes     int64      `json:"transmitBytes"`
+	LastHandshakeTime *time.Time `json:"last_handshake_time"`
+	ReceiveBytes      int64      `json:"received_bytes"`
+	TransmitBytes     int64      `json:"transmit_bytes"`
 	Endpoint          string     `json:"endpoint"`
 }

--- a/internal/storage/file.go
+++ b/internal/storage/file.go
@ -14,11 +14,15 @@ import (

 // implements Storage interface
 type FileStorage struct {
+	*InProcessWatcher
 	directory string
 }

 func NewFileStorage(directory string) *FileStorage {
-	return &FileStorage{directory}
+	return &FileStorage{
+		InProcessWatcher: NewInProcessWatcher(),
+		directory:        directory,
+	}
 }

 func (s *FileStorage) Open() error {
@ -46,6 +50,7 @@ func (s *FileStorage) Save(device *Device) error {
 	if err := ioutil.WriteFile(path, bytes, 0600); err != nil {
 		return errors.Wrapf(err, "failed to write device to file %s", path)
 	}
+	s.emitAdd(device)
 	return nil
 }

@ -113,6 +118,7 @@ func (s *FileStorage) Delete(device *Device) error {
 	if err := os.Remove(s.deviceFilePath(key(device))); err != nil {
 		return errors.Wrap(err, "failed to delete device file")
 	}
+	s.emitDelete(device)
 	return nil
 }

--- a/internal/storage/inmemory.go
+++ b/internal/storage/inmemory.go
@ -7,13 +7,15 @@ import (

 // implements Storage interface
 type InMemoryStorage struct {
+	*InProcessWatcher
 	db map[string]*Device
 }

 func NewMemoryStorage() *InMemoryStorage {
 	db := make(map[string]*Device)
 	return &InMemoryStorage{
-		db: db,
+		InProcessWatcher: NewInProcessWatcher(),
+		db:               db,
 	}
 }

@ -27,6 +29,7 @@ func (s *InMemoryStorage) Close() error {

 func (s *InMemoryStorage) Save(device *Device) error {
 	s.db[key(device)] = device
+	s.emitAdd(device)
 	return nil
 }

@ -56,5 +59,6 @@ func (s *InMemoryStorage) Get(owner string, name string) (*Device, error) {

 func (s *InMemoryStorage) Delete(device *Device) error {
 	delete(s.db, key(device))
+	s.emitDelete(device)
 	return nil
 }
--- a/internal/storage/inprocesswatcher.go
+++ b/internal/storage/inprocesswatcher.go
@ -0,0 +1,37 @@
+package storage
+
+type InProcessWatcher struct {
+	add    []Callback
+	delete []Callback
+}
+
+func NewInProcessWatcher() *InProcessWatcher {
+	return &InProcessWatcher{
+		add:    []Callback{},
+		delete: []Callback{},
+	}
+}
+
+func (w *InProcessWatcher) OnAdd(cb Callback) {
+	w.add = append(w.add, cb)
+}
+
+func (w *InProcessWatcher) OnDelete(cb Callback) {
+	w.delete = append(w.delete, cb)
+}
+
+func (w *InProcessWatcher) OnReconnect(cb func()) {
+	// noop because the inprocess watcher can't disconnect
+}
+
+func (w *InProcessWatcher) emitAdd(device *Device) {
+	for _, cb := range w.add {
+		cb(device)
+	}
+}
+
+func (w *InProcessWatcher) emitDelete(device *Device) {
+	for _, cb := range w.delete {
+		cb(device)
+	}
+}
--- a/internal/storage/pgwatcher.go
+++ b/internal/storage/pgwatcher.go
@ -0,0 +1,57 @@
+package storage
+
+import (
+	"encoding/json"
+
+	"github.com/pkg/errors"
+	"github.com/place1/pg-events/pkg/pgevents"
+	"github.com/sirupsen/logrus"
+)
+
+type PgWatcher struct {
+	*pgevents.Listener
+}
+
+func NewPgWatcher(connectionString string, table string) (*PgWatcher, error) {
+	listener, err := pgevents.OpenListener(connectionString)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to open pg listener")
+	}
+
+	if err := listener.Attach(table); err != nil {
+		return nil, errors.Wrapf(err, "failed to attach listener to table: %s", table)
+	}
+
+	return &PgWatcher{
+		Listener: listener,
+	}, nil
+}
+
+func (w *PgWatcher) OnAdd(cb Callback) {
+	w.Listener.OnEvent(func(event *pgevents.TableEvent) {
+		if event.Action == "UPDATE" || event.Action == "INSERT" {
+			w.emit(cb, event)
+		}
+	})
+}
+
+func (w *PgWatcher) OnDelete(cb Callback) {
+	w.Listener.OnEvent(func(event *pgevents.TableEvent) {
+		if event.Action == "DELETE" {
+			w.emit(cb, event)
+		}
+	})
+}
+
+func (w *PgWatcher) OnReconnect(cb func()) {
+	w.Listener.OnReconnect(cb)
+}
+
+func (w *PgWatcher) emit(cb Callback, event *pgevents.TableEvent) {
+	device := &Device{}
+	if err := json.Unmarshal([]byte(event.Data), device); err != nil {
+		logrus.Error(errors.Wrap(err, "failed to unmarshal postgres event data into device struct"))
+	} else {
+		cb(device)
+	}
+}
--- a/internal/storage/sql.go
+++ b/internal/storage/sql.go
@ -37,13 +37,14 @@ func (*GormLogger) Print(v ...interface{}) {

 // implements Storage interface
 type SQLStorage struct {
+	Watcher
 	db               *gorm.DB
 	sqlType          string
 	connectionString string
 }

 func NewSqlStorage(u *url.URL) *SQLStorage {
-	connectionString := ""
+	var connectionString string

 	switch u.Scheme {
 	case "postgres":
@ -59,6 +60,7 @@ func NewSqlStorage(u *url.URL) *SQLStorage {
 	}

 	return &SQLStorage{
+		Watcher:          nil,
 		db:               nil,
 		sqlType:          u.Scheme,
 		connectionString: connectionString,
@ -104,11 +106,23 @@ func (s *SQLStorage) Open() error {
 		return errors.Wrap(err, fmt.Sprintf("failed to connect to %s", s.sqlType))
 	}
 	s.db = db
+
 	db.SetLogger(&GormLogger{})
 	db.LogMode(true)

 	// Migrate the schema
 	s.db.AutoMigrate(&Device{})
+
+	if s.sqlType == "postgres" {
+		watcher, err := NewPgWatcher(s.connectionString, db.NewScope(&Device{}).TableName())
+		if err != nil {
+			return errors.Wrap(err, "failed to create pg watcher")
+		}
+		s.Watcher = watcher
+	} else {
+		s.Watcher = NewInProcessWatcher()
+	}
+
 	return nil
 }

--- a/scripts/run-postgres.sh
+++ b/scripts/run-postgres.sh
@ -0,0 +1,19 @@
+#!/bin/bash
+set -eou pipefail
+
+NAME="wg-access-server"
+
+if [[ ! "$(docker ps -aqf name=$NAME)" ]]; then
+  docker run \
+    -e 'POSTGRES_USER=postgres' \
+    -e 'POSTGRES_PASSWORD=example' \
+    -e 'POSTGRES_DB=postgres' \
+    -p 5432:5432 \
+    -d \
+    --name "$NAME" \
+    postgres:11-alpine
+else
+  docker start "$NAME"
+fi
+
+echo "started postgres: $NAME"