Merge branch 'feat/backend-webrtc' of beep/backend into master

.gitmodules

@@ -1,21 +1,9 @@
-[submodule "backend-bite"]
-	path = backend-bite
-	url = git@makerforce.io:beep/backend-bite.git
 [submodule "backend-core"]
 	path = backend-core
 	url = git@makerforce.io:beep/backend-core.git
-[submodule "backend-publish"]
-	path = backend-publish
-	url = git@makerforce.io:beep/backend-publish.git
-[submodule "backend-signal"]
-	path = backend-signal
-	url = git@makerforce.io:beep/backend-signal.git
 [submodule "backend-store"]
 	path = backend-store
 	url = git@makerforce.io:beep/backend-store.git
-[submodule "backend-subscribe"]
-	path = backend-subscribe
-	url = git@makerforce.io:beep/backend-subscribe.git
 [submodule "backend-transcription"]
 	path = backend-transcription
 	url = git@makerforce.io:beep/backend-transcription.git
@@ -34,3 +22,6 @@
 [submodule "backend-permissions"]
 	path = backend-permissions
 	url = git@makerforce.io:beep/backend-permissions.git
+[submodule "backend-webrtc"]
+	path = backend-webrtc
+	url = git@makerforce.io:beep/backend-webrtc.git

README.md

@@ -21,6 +21,7 @@ The microservices of Beep rely on a few background services, listed below. All o
 | `postgres` | [https://www.postgresql.org/] |
 | `redis` | [https://redis.io/] |
 | `nats` | [https://nats.io/] |
+| `minio` | [https://min.io]|
 
 ## Services
 
@@ -52,45 +53,51 @@ URL: `<base-url>/core`
 
 `core` relies on a running `postgres` instance. Is insecure if not behind `traefik` calling `auth`.
 
+### Heartbeat
+
+URL: `<base-url>/heartbeat`
+
+`heartbeat` handles "last seen" timings for users. A user pings the server periodically via a specific endpoint, which then caches the time of the ping while also updating subscribed clients. Clients subscribe through an EventSource endpoint. On first subscribe, the last cached time of the user in question is pushed to the EventSource stream.
+
+`heartbeat` relies on a running `redis` instance. Is insecure if not behind `traefik` calling `auth`.
+
+### Pictures
+
+URL: `<base-url>/pictures`
+
+`pictures` is a simple file upload server whose intended function is to just be a place to park user and group profile pictures.
+
+`pictures` relies on a running `minio` instance. Is insecure if not behind `traefik` calling `auth`.
+
+### Permissions
+
+`permissions` is an internal system meant to check a user's permission to access something. Currently uses a `user-scope` system, i.e. user-conversation. Since most things in the backend are related to conversations, the working basis of the permissions model is that if a user is in a conversation, they are pretty much good to go. Caches permissions in redis in a misguided attempt at reducing latency.
+
+`permissions` relies on a running `redis` instance.
+
 ### Bite pipeline
 
-Audio data in Beep is stored in discrete packets called "bites". The Bite pipeline takes in bites and processes them, doing things like storage and transcription to text. `publish` receives the bites, publishing them to `nats`, from which the processing services receive bite events. Output is then published again to `nats`, received by `subscribe` which pushes them as Server Sent Events.
+Audio data in Beep is stored in discrete packets called "bites". The Bite pipeline takes in bites and processes them, doing things like storage and transcription to text. Currently, in an downright terrible implementation, bites are just discrete 1400 byte chunks separated with absolutely no regard whatsoever to their content.
 
-#### `publish`
+#### `webrtc`
 
-URL: `<base-url>/publish`
+URL: `<base-url>/webrtc`
 
-`publish` accepts a POST request, containing a bite, and publishes it to NATs, to be received by services such as `bite` or `transcription`.
+`webrtc` is a WebRTC Selective Forwarding Unit (SFU) router, keeping track of which conversation a user is in and routing based on that. At the same time, it also diverts the bites to the bite pipeline and issues a store request to `store` at the same time.
 
-`publish` relies on a running `nats` instance. Is insecure if not behind `traefik` calling `auth`.
-
-#### `subscribe`
-
-URL: `<base-url>/subscribe`
-
-`subscribe` sits on the other end of the bite pipeline, waiting for responses returned by services along it. Each request pushed to the pipeline stores the user/client ID of the requester, and the client can subscribe to `subscribe` to receive the response.
-
-`subscribe` relies on a running `nats` instance. Is insecure if not behind `traefik` calling `auth`.
+`webrtc` relies on a running `nats` instance. Is insecure if not behind `traefik` calling `auth`.
 
 #### `store`
 
-`store` is a wrapper around [badger](https://github.com/dgraph-io/badger). Receives data through `nats`, generating keys based on a label supplied with the data. Also supports retrieval of specific data based on key, and scanning a range of keys based on timestamp.
+URL: `<base-url/store`
 
-`store` relies on a running `nats` instance.
+`store` is a wrapper around [badger](https://github.com/dgraph-io/badger). Receives data through `nats`, generating keys based on a label supplied with the data. Also supports retrieval of specific data based on key, and scanning a range of keys based on timestamp and supporting retrieval via HTTP endpoints.
 
-#### `bite`
-
-URL: `<base-url>/bite`
-
-`bite` stores raw audio data via `store`. Basically just acts as a forwarder to `store`, adding labels to the data and accepting HTTP requests to retrieve the data.
-
-`bite` relies on a running `nats` instance. Is insecure if not behind `traefik` calling `auth`.
+`store` relies on a running `nats` instance. Is insecure if not behind `traefik` calling `auth`.
 
 #### `transcription`
 
-URL: `<base-url>/transcription`
-
-`transcription` takes the raw audio data, packages it and then sends it to the [Google Cloud Speech-to-Text](https://cloud.google.com/speech-to-text/). Sends the transcripted result to `store` to be stored. Handles HTTP requests to retrieve transcriptions too.
+`transcription` takes the raw audio data, packages it and then sends it to the [Google Cloud Speech-to-Text](https://cloud.google.com/speech-to-text/). Sends the transcripted result to `store` to be stored.
 
 `transcription` relies on a running `nats` instance. Is insecure if not behind `traefik` calling `auth`.
 

backend-bite

@@ -1 +0,0 @@
-Subproject commit ec305353875029477f9d19d24bb829f4b1c0b9de

backend-publish

@@ -1 +0,0 @@
-Subproject commit d105441d41d01ba590c9ce4c26dfb4a763ca5125

backend-signal

@@ -1 +0,0 @@
-Subproject commit a5540dff2283b4bc7e847962403d61f37a177816

backend-store

@@ -1 +1 @@
-Subproject commit 01a6b678abfc5b42edb42abb120cd38ddc6f0754
+Subproject commit ec7026385b65e7a9cd97fe8930a9252f9be21fb6

backend-subscribe

@@ -1 +0,0 @@
-Subproject commit 5efeab4699d824722e14d7900c6b9d38675c9b93

backend-transcription

@@ -1 +1 @@
-Subproject commit b5b09fc8150d87b2953eeb758ff073462dc50aec
+Subproject commit 9e8ae9b73ed1d3dda62022d2294a07017606863d

backend-webrtc

@@ -0,0 +1 @@
+Subproject commit f7d9197940119f964d21583d8077c6fec0e1a0d3

docker-compose.yml

@@ -5,13 +5,11 @@ services:
     command: --configfile=/traefik.toml
     depends_on:
       - core
-      - signal
       - heartbeat
-      - bite
-      - publish
-      - subscribe
-      - transcription
       - auth
+      - pictures
+      - webrtc
+      - store
     ports:
       - "80:80"
       - "443:443"
@@ -76,15 +74,6 @@ services:
       - pgnet
       - traefiknet
 
-  signal:
-    build: ./backend-signal
-    tty: false
-    restart: unless-stopped
-    environment:
-      - PORT=80
-    networks:
-      - traefiknet
-
   heartbeat:
     build: ./backend-heartbeat
     tty: true
@@ -143,7 +132,6 @@ services:
     restart: unless-stopped
     depends_on:
       - minio
-      - traefik
     environment:
       - LISTEN=:80
       - MINIO_ENDPOINT=minio:9000
@@ -173,8 +161,8 @@ services:
       - permissionsnet
 
   # Pipeline
-  bite:
-    build: ./backend-bite
+  webrtc:
+    build: ./backend-webrtc
     tty: true
     restart: unless-stopped
     depends_on:
@@ -183,22 +171,8 @@ services:
       - LISTEN=:80
       - NATS=nats://nats:4222
     networks:
-      - natsnet
       - traefiknet
-
-  publish:
-    build: ./backend-publish
-    tty: true
-    restart: unless-stopped
-    depends_on:
-      - nats
-    environment:
-      - LISTEN=:80
-      - NATS=nats://nats:4222
-      - SECRET=secret
-    networks:
       - natsnet
-      - traefiknet
 
   store:
     build: ./backend-store
@@ -212,19 +186,6 @@ services:
     networks:
       - natsnet
 
-  subscribe:
-    build: ./backend-subscribe
-    tty: true
-    restart: unless-stopped
-    depends_on:
-      - nats
-    environment:
-      - LISTEN=:80
-      - NATS=nats://nats:4222
-    networks:
-      - natsnet
-      - traefiknet
-
   transcription:
     build: ./backend-transcription
     tty: true
@@ -232,12 +193,10 @@ services:
     depends_on:
       - nats
     environment:
-      - LISTEN=:80
       - NATS=nats://nats:4222
       - API_KEY=AIzaSyDxSXDefzw9gXCQaVzOCYlRn_vcC9Da9Q0
     networks:
       - natsnet
-      - traefiknet
 
 networks:
   authnet:

traefik.toml

@@ -51,22 +51,6 @@ entrypoints = ["http", "https"]
   [frontends.core.routes.one]
   rule = "PathPrefixStrip: /core/"
 
-# Signal
-[backends.signal]
-  [backends.signal.servers.one]
-  url = "http://signal"
-
-[frontends.signal]
-backend = "signal"
-entrypoints = ["http", "https"]
-  [frontends.signal.headers.customresponseheaders]
-  Access-Control-Allow-Origin = "*"
-  Access-Control-Allow-Headers = "Content-Type, Authorization, X-User-Claim"
-  Access-Control-Allow-Credentials = "true"
-  Access-Control-Allow-Methods = "GET, HEAD, POST, PUT, PATCH, DELETE"
-  [frontends.signal.routes.one]
-  rule = "PathPrefixStrip: /signal/"
-
 # Heartbeat
 [backends.heartbeat]
   [backends.heartbeat.servers.one]
@@ -83,70 +67,6 @@ entrypoints = ["http", "https"]
   [frontends.heartbeat.routes.one]
   rule = "PathPrefixStrip: /heartbeat/"
 
-# Bite
-[backends.bite]
-  [backends.bite.servers.one]
-  url = "http://bite"
-
-[frontends.bite]
-backend = "bite"
-entrypoints = ["http", "https"]
-  [frontends.bite.headers.customresponseheaders]
-  Access-Control-Allow-Origin = "*"
-  Access-Control-Allow-Headers = "Content-Type, Authorization, X-User-Claim"
-  Access-Control-Allow-Credentials = "true"
-  Access-Control-Allow-Methods = "GET, HEAD, POST, PUT, PATCH, DELETE"
-  [frontends.bite.routes.one]
-  rule = "PathPrefixStrip: /bite/"
-
-# Publish
-[backends.publish]
-  [backends.publish.servers.one]
-  url = "http://publish"
-
-[frontends.publish]
-backend = "publish"
-entrypoints = ["http", "https"]
-  [frontends.publish.headers.customresponseheaders]
-  Access-Control-Allow-Origin = "*"
-  Access-Control-Allow-Headers = "Content-Type, Authorization, X-User-Claim"
-  Access-Control-Allow-Credentials = "true"
-  Access-Control-Allow-Methods = "GET, HEAD, POST, PUT, PATCH, DELETE"
-  [frontends.publish.routes.one]
-  rule = "PathPrefixStrip: /publish/"
-
-# Subscribe
-[backends.subscribe]
-  [backends.subscribe.servers.one]
-  url = "http://subscribe"
-
-[frontends.subscribe]
-backend = "subscribe"
-entrypoints = ["http", "https"]
-  [frontends.subscribe.headers.customresponseheaders]
-  Access-Control-Allow-Origin = "*"
-  Access-Control-Allow-Headers = "Content-Type, Authorization, X-User-Claim"
-  Access-Control-Allow-Credentials = "true"
-  Access-Control-Allow-Methods = "GET, HEAD, POST, PUT, PATCH, DELETE"
-  [frontends.subscribe.routes.one]
-  rule = "PathPrefixStrip: /subscribe/"
-
-# transcription
-[backends.transcription]
-  [backends.transcription.servers.one]
-  url = "http://transcription"
-
-[frontends.transcription]
-backend = "transcription"
-entrypoints = ["http", "https"]
-  [frontends.transcription.headers.customresponseheaders]
-  Access-Control-Allow-Origin = "*"
-  Access-Control-Allow-Headers = "Content-Type, Authorization, X-User-Claim"
-  Access-Control-Allow-Credentials = "true"
-  Access-Control-Allow-Methods = "GET, HEAD, POST, PUT, PATCH, DELETE"
-  [frontends.transcription.routes.one]
-  rule = "PathPrefixStrip: /transcription/"
-
 # pictures
 [backends.pictures]
   [backends.pictures.servers.one]
@@ -163,6 +83,38 @@ entrypoints = ["http", "https"]
   [frontends.pictures.routes.one]
   rule = "PathPrefixStrip: /pictures/"
 
+# webrtc
+[backends.webrtc]
+  [backends.webrtc.servers.one]
+  url = "http://webrtc"
+
+[frontends.webrtc]
+backend = "webrtc"
+entrypoints = ["http", "https"]
+  [frontends.webrtc.headers.customresponseheaders]
+  Access-Control-Allow-Origin = "*"
+  Access-Control-Allow-Headers = "Content-Type, Authorization, X-User-Claim"
+  Access-Control-Allow-Credentials = "true"
+  Access-Control-Allow-Methods = "GET, HEAD, POST, PUT, PATCH, DELETE"
+  [frontends.webrtc.routes.one]
+  rule = "PathPrefixStrip: /webrtc/"
+
+# store
+[backends.store]
+  [backends.store.servers.one]
+  url = "http://store"
+
+[frontends.store]
+backend = "store"
+entrypoints = ["http", "https"]
+  [frontends.store.headers.customresponseheaders]
+  Access-Control-Allow-Origin = "*"
+  Access-Control-Allow-Headers = "Content-Type, Authorization, X-User-Claim"
+  Access-Control-Allow-Credentials = "true"
+  Access-Control-Allow-Methods = "GET, HEAD, POST, PUT, PATCH, DELETE"
+  [frontends.store.routes.one]
+  rule = "PathPrefixStrip: /store/"
+
 # login
 [backends.login]
   [backends.login.servers.one]