5
0
Fork 0

Basic storage functionality

master
Daniel Lim 2019-06-20 13:20:41 +08:00
parent 8b2e437ecb
commit ead40a4ff6
5 changed files with 226 additions and 1 deletions

5
.env
View File

@ -1 +1,6 @@
LISTEN=:80
MINIO_ENDPOINT=minio:9000
MINIO_ID=MINIO_ID
MINIO_KEY=MINIO_KEY
MINIO_BUCKET_NAME=beep
MINIO_LOCATION=us-east-1

View File

@ -1,3 +1,86 @@
# backend-pictures
Beep backend proxying Minio to act as a fileserver.
Beep backend proxying [Minio](https://min.io) to act as a fileserver. Need a running instance of `minio`.
**To run this service securely is to run it behind traefik forwarding auth to `backend-auth`**
## Quickstart
```
go build && ./pictures
```
## Environment Variables
Supply environment variables by either exporting them or editing `.env`.
| ENV | Definition | Default |
| --- | ---------- | ------- |
| LISTEN | Host and port number to listen on | :80 |
| MINIO_ENDPOINT | Host and port of minio | minio:9000 |
| MINIO_ID | Client id to use with minio | MINIO_ID |
| MINIO_KEY | Client key to use with minio | MINIO_KEY |
| MINIO_BUCKET_NAME | Name of bucket to store files in | beep |
| MINIO_LOCATION | Minio bucket region | us-east-1 |
## API
All requests need to be passed through `traefik` calling `backend-auth` as Forward Authentication. Otherwise, populate `X-User-Claim` with:
```json
{
"userid": "<userid>",
"clientid": "<clientid>"
}
```
### Upload FIle
```
POST /upload
```
Upload a file to be stored. Requires a request of `Content-Type` `multipart/form-data`.
#### Body
| Name | Description |
| ---- | ----------- |
| file | File to be uploaded |
#### Success (200 OK)
Name of the file as stored.
#### Errors
| Code | Description |
| ---- | ----------- |
| 400 | Error parsing file out of request |
| 500 | Error storing file in minio |
---
### Get File
```
GET /picture/:filename
```
Retrieve a picture by filename.
#### Params
| Name | Description |
| ---- | ----------- |
| filename | Name of the file to be retrieved |
#### Success (200 OK)
Image file.
#### Errors
| Code | Description |
| ---- | ----------- |
| 500 | Error retrieving file from minio |

5
go.mod
View File

@ -3,6 +3,11 @@ module pictures
go 1.12
require (
github.com/go-ini/ini v1.42.0 // indirect
github.com/joho/godotenv v1.3.0
github.com/julienschmidt/httprouter v1.2.0
github.com/minio/minio-go v6.0.14+incompatible
github.com/mitchellh/go-homedir v1.1.0 // indirect
golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 // indirect
golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b // indirect
)

17
go.sum
View File

@ -1,4 +1,21 @@
github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o=
github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 h1:IcSOAf4PyMp3U3XbIEj1/xJ2BjNN2jWv7JoyOsMxXUU=
golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b h1:lkjdUzSyJ5P1+eal9fxXX9Xg2BTfswsonKUse48C0uE=
golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

115
main.go
View File

@ -1,16 +1,27 @@
package main
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"io"
"log"
"net/http"
"os"
"strings"
"github.com/joho/godotenv"
"github.com/julienschmidt/httprouter"
"github.com/minio/minio-go"
)
const MaxBiteSize = 1024 * 1024 * 100 // 100MB
var listen string
var minioClient *minio.Client
var bucketName string
func main() {
// Load .env
@ -19,11 +30,115 @@ func main() {
log.Fatal("Error loading .env file")
}
listen = os.Getenv("LISTEN")
minioEndpoint := os.Getenv("MINIO_ENDPOINT")
minioID := os.Getenv("MINIO_ID")
minioKey := os.Getenv("MINIO_KEY")
bucketName = os.Getenv("MINIO_BUCKET_NAME")
minioLocation := os.Getenv("MINIO_LOCATION")
// Minio client
minioClient, err = minio.New(minioEndpoint, minioID, minioKey, false)
if err != nil {
log.Fatal("Error loading minio")
}
// Create bucket if it doesn't exist
err = minioClient.MakeBucket(bucketName, minioLocation)
if err != nil {
exists, err := minioClient.BucketExists(bucketName)
if err == nil && exists {
log.Printf("Bucket %s already exists", bucketName)
} else {
log.Fatal("Error creating bucket")
}
} else {
log.Printf("Created bucket %s", bucketName)
}
// Routes
router := httprouter.New()
router.POST("/upload", AuthMiddleware(Upload))
router.GET("/picture/:filename", GetFile)
// Start server
log.Printf("starting server on %s", listen)
log.Fatal(http.ListenAndServe(listen, router))
}
// Pull Auth header
type RawClient struct {
UserId string `json:"userid"`
ClientId string `json:"clientid"`
}
func AuthMiddleware(next httprouter.Handle) httprouter.Handle {
return func (w http.ResponseWriter, r *http.Request, p httprouter.Params) {
ua := r.Header.Get("X-User-Claim")
if ua == "" {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
var client RawClient
err := json.Unmarshal([]byte(ua), &client)
if err != nil {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
context := context.WithValue(r.Context(), "user", client)
next(w, r.WithContext(context), p)
}
}
func Upload(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
client := r.Context().Value("user").(RawClient)
var buf bytes.Buffer
file, header, err := r.FormFile("file")
if err != nil {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
defer file.Close()
originalName := strings.Split(header.Filename, ".")
io.Copy(&buf, file)
reader := bytes.NewReader(buf.Bytes())
fileName := RandomHex() + "." + originalName[1]
options := minio.PutObjectOptions{
UserMetadata: make(map[string] string),
}
options.UserMetadata["owner"] = client.UserId
_, err = minioClient.PutObject(bucketName, fileName, reader, header.Size, options)
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
io.WriteString(w, fileName)
}
func RandomHex() string {
b := make([]byte, 16)
_, err := rand.Read(b)
if err != nil {
panic("unable to generate 16 bits of randomness")
}
return hex.EncodeToString(b)
}
func GetFile(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
fileName := p.ByName("filename")
options := minio.GetObjectOptions{}
reader, err := minioClient.GetObject(bucketName, fileName, options)
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
io.Copy(w, reader)
}