Skip to content

Commit 04496b6

Browse files
authored
Merge 1.3 (#13)
* Rewrite Dockerfile * Fix dev Dockerfile * Update dockerfiles and configuration accessor * Add more test suites * Add gatling tests * Enhance cache speed and tests * Test coverage over 60% * Improve cache speed and post results * Implement by regex configuration * Refactoring and performance update done * Update results * Upgrades * Introduce Ristretto cache module * Increase performance by 43 percent * Close unused channels * GH action setup on tag creation * Update docker-compose for prod * Generate artifacts * Update README * Update docker-compose files * Update configuration.sample * Add package builder/releaser
1 parent 7893e39 commit 04496b6

29 files changed

Lines changed: 1087 additions & 225 deletions

.github/workflows/release.yml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: Build container and publish to docker hub
2+
3+
on:
4+
create:
5+
tags: ["v*"]
6+
7+
jobs:
8+
generate-binary:
9+
name: Generate binary
10+
runs-on: ubuntu-latest
11+
steps:
12+
-
13+
name: Get tag name
14+
run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/}
15+
-
16+
name: Checkout code
17+
uses: actions/checkout@v2
18+
with:
19+
fetch-depth: 0
20+
-
21+
name: Set up QEMU
22+
uses: docker/setup-qemu-action@v1
23+
-
24+
name: Set up Docker Buildx
25+
uses: docker/setup-buildx-action@v1
26+
-
27+
name: Login to DockerHub
28+
uses: docker/login-action@v1
29+
with:
30+
username: ${{ secrets.DOCKER_USERNAME }}
31+
password: ${{ secrets.DOCKER_PASSWORD }}
32+
-
33+
name: Build & push Docker image containing only binary
34+
id: docker_build
35+
uses: docker/build-push-action@v2
36+
with:
37+
push: true
38+
file: ./Dockerfile-prod
39+
tags: |
40+
darkweak/souin:latest
41+
darkweak/souin:${{ env.RELEASE_VERSION }}
42+
generate-artifacts:
43+
name: Generate cross-platform builds
44+
runs-on: ubuntu-latest
45+
steps:
46+
-
47+
name: Set up Go
48+
uses: actions/setup-go@v2
49+
with:
50+
go-version: 1.13
51+
-
52+
name: Checkout
53+
uses: actions/checkout@v2
54+
with:
55+
fetch-depth: 0
56+
-
57+
name: Run GoReleaser
58+
uses: goreleaser/goreleaser-action@v2
59+
with:
60+
version: latest
61+
args: release --rm-dist
62+
env:
63+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Dockerfile-dev

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,15 @@
1-
FROM golang:1.13-alpine
1+
FROM golang:1.13-alpine AS souin
22

33
RUN apk update && apk upgrade && \
44
apk add --no-cache bash git openssh gcc libc-dev
5+
ENV GOPATH /app
6+
RUN go get -u \
7+
golang.org/x/lint/golint \
8+
github.com/allegro/bigcache \
9+
github.com/dgraph-io/ristretto \
10+
github.com/fsnotify/fsnotify \
11+
github.com/go-redis/redis \
12+
gopkg.in/yaml.v2
513

614
RUN mkdir -p /app/src/github.com/darkweak/souin
715
RUN mkdir -p /ssl
@@ -10,16 +18,10 @@ ADD ./cache /app/src/github.com/darkweak/souin/cache
1018
ADD ./errors /app/src/github.com/darkweak/souin/errors
1119
ADD ./providers /app/src/github.com/darkweak/souin/providers
1220
ADD ./default/server.* /app/src/github.com/darkweak/souin/
13-
ADD ./configuration/* /app/src/github.com/darkweak/souin/configuration/
21+
ADD ./configuration/configuration.yml /
22+
ADD ./configuration /app/src/github.com/darkweak/souin/configuration
1423

1524
WORKDIR /app/src/github.com/darkweak/souin
16-
ENV GOPATH /app
17-
RUN go get -u \
18-
golang.org/x/lint/golint \
19-
github.com/allegro/bigcache \
20-
github.com/fsnotify/fsnotify \
21-
github.com/go-redis/redis \
22-
gopkg.in/yaml.v2
2325

2426
EXPOSE 80
2527

Dockerfile-prod

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,43 @@
1-
FROM golang:1.13-alpine
1+
FROM golang:1.13-alpine AS builder
22

33
RUN apk update && apk upgrade && \
44
apk add --no-cache bash git openssh gcc libc-dev
5+
ENV GOPATH /app
6+
ENV GOOS linux
7+
ENV GOARCH arm64
8+
RUN go get -u \
9+
golang.org/x/lint/golint \
10+
github.com/allegro/bigcache \
11+
github.com/dgraph-io/ristretto \
12+
github.com/fsnotify/fsnotify \
13+
github.com/go-redis/redis \
14+
gopkg.in/yaml.v2
515

616
RUN mkdir -p /app/src/github.com/darkweak/cmd
7-
RUN mkdir -p /ssl
817
RUN mkdir -p /app/src/github.com/darkweak/souin
18+
RUN mkdir -p /ssl
919
ADD ./*.go /app/src/github.com/darkweak/souin/
1020
ADD ./cache /app/src/github.com/darkweak/souin/cache
1121
ADD ./errors /app/src/github.com/darkweak/souin/errors
1222
ADD ./providers /app/src/github.com/darkweak/souin/providers
1323
ADD ./default/server.* /app/src/github.com/darkweak/souin/
24+
ADD ./configuration/configuration.yml /
1425
ADD ./configuration/* /app/src/github.com/darkweak/souin/configuration/
15-
ADD ./entrypoint.sh /app/src/github.com/darkweak/souin/entrypoint.sh
1626

1727
WORKDIR /app/src/github.com/darkweak/souin
18-
ENV GOPATH /app
19-
ENV GOOS linux
20-
ENV GOARCH arm
21-
RUN go get -u \
22-
golang.org/x/lint/golint \
23-
github.com/allegro/bigcache \
24-
github.com/fsnotify/fsnotify \
25-
github.com/go-redis/redis \
26-
gopkg.in/yaml.v2
27-
RUN chmod 755 ./entrypoint.sh
28+
29+
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a \
30+
-tags netgo -ldflags '-w -extldflags "-static"' -o /app/cmd/souin .
2831

2932
EXPOSE 80
3033

31-
ENTRYPOINT ["./entrypoint.sh"]
34+
FROM alpine:latest AS souin
35+
36+
COPY --from=builder /app/cmd/souin .
37+
COPY --from=builder /app/src/github.com/darkweak/souin/configuration .
38+
COPY --from=builder /ssl/ .
39+
COPY --from=builder /app/src/github.com/darkweak/souin/server.crt .
40+
COPY --from=builder /app/src/github.com/darkweak/souin/server.key .
41+
RUN chmod +x ./souin
42+
43+
CMD ["./souin"]

Makefile

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
.PHONY: build-app build-dev create-network down env-dev env-prod help lint tests up validate
1+
.PHONY: build-app build-dev coverage create-network down env-dev env-prod gatling generate-plantUML help lint log tests up validate
22

33
DC=docker-compose
44
DC_BUILD=$(DC) build
@@ -12,6 +12,10 @@ build-dev: env-dev ## Build containers with dev env vars
1212
$(DC_BUILD) souin
1313
$(MAKE) up
1414

15+
coverage: ## Show code coverage
16+
$(DC_EXEC) souin go test ./... -coverprofile cover.out
17+
$(DC_EXEC) souin go tool cover -func cover.out
18+
1519
create-network: ## Create network
1620
docker network create your_network
1721

@@ -26,6 +30,9 @@ env-prod: ## Up container with prod env vars
2630
cp Dockerfile-prod Dockerfile
2731
cp docker-compose.yml.prod docker-compose.yml
2832

33+
gatling: ## Launch gatling scenarios
34+
cd ./gatling && $(DC) up
35+
2936
generate-plantUML: ## Generate plantUML diagrams
3037
cd ./docs/plantUML && sh generate.sh && cd ../..
3138

README.md

Lines changed: 52 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,10 @@
1111
5. [Examples](#examples)
1212
5.1. [Træfik container](#træfik-container)
1313
6. [SSL](#ssl)
14-
6.1. [Træfik](#træfik)
14+
6.1. [Træfik](#træfik)
15+
6.2. [Apache](#apache)
16+
6.3. [Nginx](#nginx)
17+
6. [Credits](#credits)
1518

1619
[![Travis CI](https://travis-ci.com/Darkweak/Souin.svg?branch=master)](https://travis-ci.com/Darkweak/Souin)
1720

@@ -22,49 +25,72 @@ Souin is a new cache system suitable for every reverse-proxy. It will be placed
2225
As it's written in go, it can be deployed on any server and thanks to the docker integration, it will be easy to install on top of a Swarm or a kubernetes instance.
2326

2427
## Configuration
25-
The configuration file is stored at `configuration/configuration.yml`. You can edit it provided you fill at least the required parameters as shown below.
28+
The configuration file is stored at `/anywhere/configuration.yml`. You can edit it provided you fill at least the required parameters as shown below.
2629

2730
### Required configuration
2831
```yaml
29-
ttl: 100 #TTL in second
30-
reverse_proxy_url: 'http://traefik' # The reverse-proxy http address
31-
cache:
32-
port:
32+
default_cache: # Required part
33+
port: # Ports to expose Souin
3334
web: 80
3435
tls: 443
36+
ttl: 10 # Default TTL
37+
reverse_proxy_url: 'http://traefik' # If it's in the same network you can use http://your-service. Then just use https://yourdomain.com
3538
```
3639
This is a fully working minimal configuration for a Souin instance
3740
3841
| Key | Description | Value example |
3942
|:---:|:---:|:---:|
40-
|`ttl`|Duration to cache request (in seconds)|10|
41-
|`reverse_proxy_url`|The reverse-proxy's instance URL (Apache, Nginx, Træfik...)|- `http://yourservice` (Container way)<br/>`http://localhost:81` (Local way)|
42-
|`cache.port.{web,tls}`|The device's local HTTP/TLS port that Souin should be listening on |Respectively `80` and `443`|
43+
|`default_cache.port.{web,tls}`|The device's local HTTP/TLS port that Souin should be listening on |Respectively `80` and `443`|
44+
|`default_cache.ttl`|Duration to cache request (in seconds)|10|
45+
|`reverse_proxy_url`|The reverse-proxy's instance URL (Apache, Nginx, Træfik...)|- `http://yourservice` (Container way)<br/>`http://localhost:81` (Local way)<br/>`http://yourdomain.com:81` (Network way)|
4346

4447
### Optional configuration
4548
```yaml
46-
redis:
47-
url: 'redis:6379' # Redis http address, only used for redis provider
48-
regex:
49-
exclude: 'ARegexHere'
50-
ssl_providers: # Must match your volumes to /ssl/{provider}.json
49+
# /anywhere/configuration.yml
50+
default_cache:
51+
headers: # Default headers concatenated in stored keys
52+
- Authorization
53+
providers:
54+
- all # Enable all providers by default
55+
redis: # Redis configuration
56+
url: 'redis:6379'
57+
regex:
58+
exclude: 'ARegexHere' # Regex to exclude from cache
59+
ssl_providers: # The {providers}.json to use
5160
- traefik
52-
cache:
53-
headers:
54-
- Authorization # Can be any other headers
55-
providers: # By default it will use in-memory and redis cache. It can be either `all`, `redis` or `memory`.
56-
- all # Can be set to all if you want to enable all providers instead of specifying each one
57-
# - memory
58-
# - redis
61+
urls:
62+
'https:\/\/domain.com\/first-.+': # First regex route configuration
63+
ttl: 1000 # Override default TTL
64+
providers: # Providers list (each item must be in default providers list)
65+
- redis
66+
- memory
67+
- ristretto
68+
'https:\/\/domain.com\/second-route': # Second regex route configuration
69+
ttl: 10 # Override default TTL
70+
headers: # Override default headers
71+
- Authorization
72+
providers: # Providers list (each item must be in default providers list)
73+
- memory
74+
'https?:\/\/mysubdomain\.domain\.com': # Third regex route configuration
75+
ttl: 50
76+
headers: # Override default headers
77+
- Authorization
78+
- 'Content-Type'
79+
providers: # Providers list (each item must be in default providers list)
80+
- redis
5981
```
6082

6183
| Key | Description | Value example |
6284
|:---:|:---:|:---:|
63-
|`redis.url`|The redis url, used if you enabled it in the provider section|`redis:6379` (container way) and `http://yourdomain.com:6379` (network way)|
64-
|`regex.exclude`|The regex used to prevent paths being cached|`^[A-z]+.*$`|
85+
|`default_cache.headers`|List of headers to include to the cache|`- Authorization`<br/><br/>`- Content-Type`<br/><br/>`- X-Additional-Header`|
86+
|`default_cache.providers`|Your providers list to cache your data, by default it will use all systems|`- all`<br/><br/>`- memory`<br/><br/>`- redis`|
87+
|`default_cache.redis.url`|The redis url, used if you enabled it in the provider section|`redis:6379` (container way) and `http://yourdomain.com:6379` (network way)|
88+
|`default_cache.regex.exclude`|The regex used to prevent paths being cached|`^[A-z]+.*$`|
6589
|`ssl_providers`|List of your providers handling certificates|`- traefik`<br/><br/>`- nginx`<br/><br/>`- apache`|
66-
|`cache.headers`|List of headers to include to the cache|`- Authorization`<br/><br/>`- Content-Type`<br/><br/>`- X-Additional-Header`|
67-
|`cache.providers`|Your providers list to cache your data, by default it will use all systems|`- all`<br/><br/>`- memory`<br/><br/>`- redis`|
90+
|`urls.{your url or regex}`|List of your custom configuration depending each URL or regex|'https:\/\/yourdomain.com'|
91+
|`urls.{your url or regex}.ttl`|Override the default TTL if defined|99999|
92+
|`urls.{your url or regex}.headers`|Override the default headers if defined|`- Authorization`<br/><br/>`- 'Content-Type'`|
93+
|`urls.{your url or regex}.providers`|Override the default providers if defined|`- redis`<br/><br/>`- ristretto`|
6894

6995
## Diagrams
7096

@@ -123,8 +149,7 @@ x-networks: &networks
123149
124150
services:
125151
souin:
126-
build:
127-
context: .
152+
image: darkweak/souin:latest
128153
ports:
129154
- 80:80
130155
- 443:443
@@ -133,7 +158,6 @@ services:
133158
environment:
134159
GOPATH: /app
135160
volumes:
136-
- ./cmd:/app/cmd
137161
- /anywhere/traefik.json:/ssl/traefik.json
138162
<<: *networks
139163

cache/providers/abstract.go

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,14 @@ import (
1010
// AbstractProviderInterface should be implemented in any providers
1111
type AbstractProviderInterface interface {
1212
GetRequestInCache(key string) types.ReverseResponse
13-
SetRequestInCache(key string, value []byte)
13+
SetRequestInCache(key string, value []byte, url configuration.URL)
1414
DeleteRequestInCache(key string)
15-
Init()
15+
Init() error
1616
}
1717

18-
// PathnameNotInRegex check if pathname is in parameter regex var
19-
func PathnameNotInRegex(pathname string, configuration configuration.Configuration) bool {
20-
b, _ := regexp.Match(configuration.Regex.Exclude, []byte(pathname))
18+
// PathnameNotInExcludeRegex check if pathname is in parameter regex var
19+
func PathnameNotInExcludeRegex(pathname string, configuration configuration.Configuration) bool {
20+
b, _ := regexp.Match(configuration.DefaultCache.Regex.Exclude, []byte(pathname))
2121
return !b
2222
}
2323

@@ -31,19 +31,24 @@ func contains(s []string, e string) bool {
3131
}
3232

3333
// InitializeProviders allow to generate the providers array according to the configuration
34-
func InitializeProviders(configuration configuration.Configuration) *[]AbstractProviderInterface {
35-
var providers []AbstractProviderInterface
34+
func InitializeProviders(configuration configuration.Configuration) map[string]AbstractProviderInterface {
35+
providers := make(map[string]AbstractProviderInterface)
3636

37-
if len(configuration.Cache.Providers) == 0 || contains(configuration.Cache.Providers, "all") {
38-
providers = append(providers, MemoryConnectionFactory(configuration), RedisConnectionFactory(configuration))
37+
if len(configuration.DefaultCache.Providers) == 0 || contains(configuration.DefaultCache.Providers, "all") {
38+
providers["memory"] = MemoryConnectionFactory(configuration)
39+
providers["redis"] = RedisConnectionFactory(configuration)
40+
providers["ristretto"] = RistrettoConnectionFactory(configuration)
3941
} else {
40-
if contains(configuration.Cache.Providers, "redis") {
41-
providers = append(providers, RedisConnectionFactory(configuration))
42+
if contains(configuration.DefaultCache.Providers, "redis") {
43+
providers["redis"] = RedisConnectionFactory(configuration)
4244
}
43-
if contains(configuration.Cache.Providers, "memory") {
44-
providers = append(providers, MemoryConnectionFactory(configuration))
45+
if contains(configuration.DefaultCache.Providers, "memory") {
46+
providers["memory"] = MemoryConnectionFactory(configuration)
47+
}
48+
if contains(configuration.DefaultCache.Providers, "ristretto") {
49+
providers["ristretto"] = RistrettoConnectionFactory(configuration)
4550
}
4651
}
4752

48-
return &providers
53+
return providers
4954
}

0 commit comments

Comments
 (0)