Multiple SSL files, dynamic SSL, works with Træfik provider (#2)

* Fix blank data returned and setup for multi ssl with providers

* Enhance multiple certificates with providers (Træfik)

* RC for 1.0 - Manage multi SSL done

* Fix CI

* Fix CI

* Add Header response initialization

* Add dynamic certs

* Update doc

* Fix travis

* Fix linter

* Makefile

* Handle acme.json doesn't exist or an error occurred

Author: darkweak

Dockerfile-dev

@@ -6,6 +6,7 @@ RUN apk update && apk upgrade && \
 RUN mkdir -p /app/src/github.com/darkweak/souin
 ADD ./*.go /app/src/github.com/darkweak/souin/
 ADD ./cache /app/src/github.com/darkweak/souin/cache
+ADD ./providers /app/src/github.com/darkweak/souin/providers
 ADD ./default/server.* /app/src/github.com/darkweak/souin/
 
 WORKDIR /app/src/github.com/darkweak/souin

Dockerfile-prod

@@ -7,6 +7,7 @@ RUN mkdir -p /app/src/github.com/darkweak/cmd
 RUN mkdir -p /app/src/github.com/darkweak/souin
 ADD ./*.go /app/src/github.com/darkweak/souin/
 ADD ./cache /app/src/github.com/darkweak/souin/cache
+ADD ./providers /app/src/github.com/darkweak/souin/providers
 ADD ./default/server.* /app/src/github.com/darkweak/souin/
 ADD ./entrypoint.sh /app/src/github.com/darkweak/souin/entrypoint.sh
 

Makefile

@@ -32,7 +32,7 @@ help:
 	@grep -E '(^[0-9a-zA-Z_-]+:.*?##.*$$)|(^##)' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[32m%-25s\033[0m %s\n", $$1, $$2}' | sed -e 's/\[32m##/[33m/'
 
 lint: ## Run lint
-	$(DC_EXEC) souin /app/bin/golint ./cache
+	$(DC_EXEC) souin /app/bin/golint ./...
 
 tests: ## Run tests
 	$(DC_EXEC) souin go test -v ./...

README.md

@@ -101,6 +101,7 @@ services:
       GOPATH: /app
     volumes:
       - ./cmd:/app/cmd
+      - ./acme.json:/app/src/github.com/darkweak/souin/acme.json
     <<: *networks
 
   redis:
@@ -111,3 +112,15 @@ networks:
   your_network:
     external: true
 ```
+
+## SSL
+
+### Træfik
+As Souin is compatible with Træfik, it can use (and it should use) acme.json provided on træfik. Souin will get new/updated certs from Træfik, then your SSL certs will be up to date as far as Træfik will be too
+To provide, acme, use just have to map volume as above
+```yaml
+    volumes:
+      - /anywhere/acme.json:/app/src/github.com/darkweak/souin/acme.json
+```
+At the moment you can't choose the path for the acme.json in the container, in the future you'll be able to do that just setting one env var
+If none acme.json is provided to container, a default cert will be served.

cache/redisConnection.go

@@ -21,8 +21,7 @@ func pathnameNotInRegex(pathname string) bool {
 	return !b
 }
 
-func getRequestInCache(pathname string) ReverseResponse {
-	client := redisClientConnectionFactory()
+func getRequestInCache(pathname string, client *redis.Client) ReverseResponse {
 	val2, err := client.Get(pathname).Result()
 
 	if err != nil {
@@ -32,20 +31,17 @@ func getRequestInCache(pathname string) ReverseResponse {
 	return ReverseResponse{val2, nil, nil}
 }
 
-func deleteKey(key string) {
-	client := redisClientConnectionFactory();
+func deleteKey(key string, client *redis.Client) {
 	client.Do("del", key)
 }
 
-func deleteKeys(regex string) {
-	client := redisClientConnectionFactory();
+func deleteKeys(regex string, client *redis.Client) {
 	for _, i := range client.Keys(regex).Val() {
 		client.Do("del", i)
 	}
 }
 
-func setRequestInCache(pathname string, data []byte) {
-	client := redisClientConnectionFactory()
+func setRequestInCache(pathname string, data []byte, client *redis.Client) {
 	value, _ := strconv.Atoi(os.Getenv("TTL"))
 
 	err := client.Set(pathname, string(data), time.Duration(value) * time.Second).Err()

cache/requestService.go

@@ -8,6 +8,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"strings"
+	"github.com/go-redis/redis"
+	"strconv"
 )
 
 //RequestResponse object contains the request belongs to reverse-proxy
@@ -38,20 +40,23 @@ func getKeyFromResponse(resp *http.Response) string {
 	return resp.Request.Host + resp.Request.URL.Path
 }
 
-func rewriteBody(resp *http.Response) (err error) {
+func rewriteBody(resp *http.Response, redisClient *redis.Client) (err error) {
 	b := bytes.Replace(commonLoadingRequest(resp), []byte("server"), []byte("schmerver"), -1)
 	body := ioutil.NopCloser(bytes.NewReader(b))
+	resp.Header = make(http.Header)
+	resp.Body = body
+	resp.ContentLength = int64(len(b))
+	resp.Header.Set("Content-Length", strconv.Itoa(len(b)))
 
 	if pathnameNotInRegex(resp.Request.Host+resp.Request.URL.Path) && !hasNotAllowedHeaders(resp) && nil == resp.Request.Context().Err() {
-		if http.MethodGet == resp.Request.Method {
+		key := getKeyFromResponse(resp)
+		if http.MethodGet == resp.Request.Method && len(b) > 0 {
 			r, _ := json.Marshal(RequestResponse{b, resp.Header})
-
 			go func() {
-				setRequestInCache(getKeyFromResponse(resp), r)
+				setRequestInCache(key, r, redisClient)
 			}()
 		} else {
-			key := getKeyFromResponse(resp)
-			deleteKey(key)
+			deleteKey(key, redisClient)
 
 			if http.MethodDelete == resp.Request.Method || http.MethodPut == resp.Request.Method || http.MethodPatch == resp.Request.Method {
 				newKeySplitted := strings.Split(key, "/")
@@ -63,25 +68,28 @@ func rewriteBody(resp *http.Response) (err error) {
 						newKey += "/"
 					}
 				}
-				deleteKey(newKey)
+				go func() {
+					deleteKey(newKey, redisClient)
+				}()
 			}
 		}
 	}
 
-	resp.Body = body
 	return nil
 }
 
-func requestReverseProxy(req *http.Request, url *url.URL) ReverseResponse  {
+func requestReverseProxy(req *http.Request, url *url.URL, redisClient *redis.Client) ReverseResponse  {
 	req.URL.Host = req.Host
 	req.URL.Scheme = url.Scheme
 	req.Header.Set("X-Forwarded-Host", req.Header.Get("Host"))
 
 	proxy := httputil.NewSingleHostReverseProxy(url)
-	proxy.ModifyResponse = rewriteBody
+	proxy.ModifyResponse = func(response *http.Response) error {
+		return rewriteBody(response, redisClient)
+	}
 
 	return ReverseResponse{
-		"",
+		"bad",
 		proxy,
 		req,
 	}

cache/requestService_test.go

@@ -15,6 +15,10 @@ import (
 const DOMAIN = "domain.com"
 const PATH = "/testing"
 
+func mockRedis() *redis.Client {
+	return redisClientConnectionFactory()
+}
+
 func mockResponse(path string, method string, body string, code int) *http.Response {
 	return &http.Response{
 		Status:           "",
@@ -82,7 +86,7 @@ func shouldNotHaveKey(pathname string) bool {
 
 func TestKeyShouldBeDeletedOnPost(t *testing.T) {
 	populateRedisWithFakeData()
-	rewriteBody(mockResponse(PATH, http.MethodPost, "My second response", 201))
+	rewriteBody(mockResponse(PATH, http.MethodPost, "My second response", 201), mockRedis())
 	time.Sleep(10 * time.Second)
 	if !shouldNotHaveKey(PATH) {
 		generateError(t, "The key "+DOMAIN+PATH+" shouldn't exist.")
@@ -101,12 +105,12 @@ func verifyKeysExists(t *testing.T, path string, keys []string) {
 
 func TestKeyShouldBeDeletedOnPut(t *testing.T) {
 	populateRedisWithFakeData()
-	rewriteBody(mockResponse(PATH+"/1", http.MethodPut, "My second response", 200))
+	rewriteBody(mockResponse(PATH+"/1", http.MethodPut, "My second response", 200), mockRedis())
 	verifyKeysExists(t, PATH, []string{"", "/1"})
 }
 
 func TestKeyShouldBeDeletedOnDelete(t *testing.T) {
 	populateRedisWithFakeData()
-	rewriteBody(mockResponse(PATH+"/1", http.MethodDelete, "", 200))
+	rewriteBody(mockResponse(PATH+"/1", http.MethodDelete, "", 200), mockRedis())
 	verifyKeysExists(t, PATH, []string{"", "/1"})
 }

cache/souin.go

@@ -6,23 +6,29 @@ import (
 	"os"
 	"net/http/httputil"
 	"encoding/json"
+	"github.com/go-redis/redis"
+	"crypto/tls"
+	"github.com/darkweak/souin/providers"
+	"fmt"
+	"context"
+	"net"
 )
 
 // ReverseResponse object contains the response from reverse-proxy
 type ReverseResponse struct {
 	response string
-	proxy *httputil.ReverseProxy
-	request *http.Request
+	proxy    *httputil.ReverseProxy
+	request  *http.Request
 }
 
-func serveReverseProxy(res http.ResponseWriter, req *http.Request) {
+func serveReverseProxy(res http.ResponseWriter, req *http.Request, redisClient *redis.Client) {
 	url, _ := url.Parse(os.Getenv("REVERSE_PROXY"))
 	ctx := req.Context()
 
 	responses := make(chan ReverseResponse)
 	go func() {
-		responses<- getRequestInCache(req.Host + req.URL.Path)
-		responses<- requestReverseProxy(req, url)
+		responses <- getRequestInCache(req.Host+req.URL.Path, redisClient)
+		responses <- requestReverseProxy(req, url, redisClient)
 	}()
 
 	response := <-responses
@@ -33,6 +39,7 @@ func serveReverseProxy(res http.ResponseWriter, req *http.Request) {
 		if err != nil {
 			panic(err)
 		}
+
 		for k, v := range responseJSON.Headers {
 			res.Header().Set(k, v[0])
 		}
@@ -44,15 +51,67 @@ func serveReverseProxy(res http.ResponseWriter, req *http.Request) {
 	}
 }
 
+func startServer(tlsconfig *tls.Config) (net.Listener, *http.Server) {
+	tlsconfig.BuildNameToCertificate()
+	server := http.Server{
+		Addr:      ":443",
+		Handler:   nil,
+		TLSConfig: tlsconfig,
+	}
+	listener, err := tls.Listen("tcp", ":443", tlsconfig)
+	if err != nil {
+		fmt.Println(err)
+	}
+	go func() {
+		error := server.Serve(listener)
+		fmt.Println("YO")
+		fmt.Println(error)
+		fmt.Println("LO")
+		if nil != error {
+			fmt.Println(error)
+		}
+	}()
+
+	return listener, &server
+}
+
 // Start cache system
 func Start() {
-	http.HandleFunc("/", serveReverseProxy)
+	redisClient := redisClientConnectionFactory()
+	configChannel := make(chan int)
+	tlsconfig := &tls.Config{
+		Certificates: make([]tls.Certificate, 0),
+		NameToCertificate: make(map[string]*tls.Certificate),
+		InsecureSkipVerify: true,
+	}
+	v, _ := tls.LoadX509KeyPair("server.crt", "server.key")
+	tlsconfig.Certificates = append(tlsconfig.Certificates, v)
+	certificates := providers.CommonProvider{
+		Certificates: make(map[string]providers.Certificate),
+	}
+
 	go func() {
-		if err := http.ListenAndServeTLS(":" + os.Getenv("CACHE_TLS_PORT"), "server.crt", "server.key", nil); err != nil {
-			panic(err)
+		providers.InitProviders(&certificates, tlsconfig, &configChannel)
+	}()
+
+	http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
+		serveReverseProxy(writer, request, redisClient)
+	})
+	go func() {
+		listener, server := startServer(tlsconfig)
+		for {
+			select {
+			case <- configChannel:
+				listener.Close()
+				if err := server.Shutdown(context.Background()); err != nil {
+					fmt.Errorf("Shutdown failed: %s", err)
+				}
+				listener, server = startServer(tlsconfig)
+			}
 		}
+
 	}()
-	if err := http.ListenAndServe(":" + os.Getenv("CACHE_PORT"), nil); err != nil {
+	if err := http.ListenAndServe(":"+os.Getenv("CACHE_PORT"), nil); err != nil {
 		panic(err)
 	}
 

docker-compose.yml.prod

@@ -23,6 +23,7 @@ services:
       GOPATH: /app
     volumes:
       - ./cmd:/app/cmd
+      - ./acme.json:/app/src/github.com/darkweak/souin/acme.json
     <<: *networks
 
   redis:

go.mod

@@ -2,4 +2,8 @@ module github.com/darkweak/souin
 
 go 1.13
 
-require github.com/go-redis/redis v6.15.6+incompatible
+require (
+	github.com/fsnotify/fsnotify v1.4.7
+	github.com/go-redis/redis v6.15.6+incompatible
+	golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e // indirect
+)