Merge pull request #2373 from knadh/feat-visual-editor

This is a mega patch with a massive number of significant, complex
changes integrating the visual, drag-and-drop block editor library,
email-builder: https://github.com/usewaypoint/email-builder-js

It adds a new template type (`visual`) to templates and integrates the editor
UI in Admin -> Templates and Admin -> Campaigns UIs. There are changes to
templates and campaign APIs, UI flows, and database table schemas.

`email-builder` is written in TypeScript and React (eww) and is a massive lib,
(2.3MB minified+gzipped), bigger than all of listmonk's own JS and other deps
combined, but alas, there is no other viable editor and it has been a popular
demand for a long time.

So, `email-builder` is built separately, and the static assets are copied along
with listmonk's assets. Since it's basically a separate "app" altogether,
it's loaded as an iframe on the editor pages, outside of the Vue app.

The visual editor integration is courtesy of @vividvilla who did all the
R&D and trial and error and sent the original PR to make this possible.

I really loathe the Javascript ecosystem. Sigh.

Author: knadh

.gitignore

@@ -2,9 +2,16 @@ frontend/node_modules/
 frontend/.cache/
 frontend/yarn.lock
 frontend/build/
+frontend/public/static/email-builder/
+frontend/dist/
+email-builder/node_modules/
+email-builder/.cache/
+email-builder/yarn.lock
+email-builder/dist/
 .vscode/
 
 config.toml
 node_modules
 listmonk
-dist/*
+dist/*
+uploads/

Makefile

@@ -11,14 +11,26 @@ GOPATH ?= $(HOME)/go
 STUFFBIN ?= $(GOPATH)/bin/stuffbin
 FRONTEND_YARN_MODULES = frontend/node_modules
 FRONTEND_DIST = frontend/dist
+FRONTEND_EMAIL_BUILDER_DIST_FINAL = frontend/public/static/email-builder
 FRONTEND_DEPS = \
 	$(FRONTEND_YARN_MODULES) \
+	$(FRONTEND_EMAIL_BUILDER_DIST_FINAL) \
 	frontend/index.html \
 	frontend/package.json \
 	frontend/vite.config.js \
 	frontend/.eslintrc.js \
 	$(shell find frontend/fontello frontend/public frontend/src -type f)
 
+FRONTEND_EMAIL_BUILDER = frontend/email-builder
+FRONTEND_EMAIL_BUILDER_YARN_MODULES = $(FRONTEND_EMAIL_BUILDER)/node_modules
+FRONTEND_EMAIL_BUILDER_DIST = $(FRONTEND_EMAIL_BUILDER)/dist
+FRONTEND_EMAIL_BUILDER_DEPS = \
+	$(FRONTEND_EMAIL_BUILDER_YARN_MODULES) \
+	$(FRONTEND_EMAIL_BUILDER)/package.json \
+	$(FRONTEND_EMAIL_BUILDER)/tsconfig.json \
+	$(FRONTEND_EMAIL_BUILDER)/vite.config.ts \
+	$(shell find $(FRONTEND_EMAIL_BUILDER)/src -type f)
+
 BIN := listmonk
 STATIC := config.toml.sample \
 	schema.sql queries.sql permissions.json \
@@ -37,6 +49,10 @@ $(FRONTEND_YARN_MODULES): frontend/package.json frontend/yarn.lock
 	cd frontend && $(YARN) install
 	touch -c $(FRONTEND_YARN_MODULES)
 
+$(FRONTEND_EMAIL_BUILDER_YARN_MODULES): frontend/package.json frontend/yarn.lock
+	cd $(FRONTEND_EMAIL_BUILDER) && $(YARN) install
+	touch -c $(FRONTEND_EMAIL_BUILDER_YARN_MODULES)
+
 # Build the backend to ./listmonk.
 $(BIN): $(shell find . -type f -name "*.go") go.mod go.sum schema.sql queries.sql permissions.json
 	CGO_ENABLED=0 go build -o ${BIN} -ldflags="-s -w -X 'main.buildString=${BUILDSTR}' -X 'main.versionString=${VERSION}'" cmd/*.go
@@ -51,13 +67,26 @@ $(FRONTEND_DIST): $(FRONTEND_DEPS)
 	export VUE_APP_VERSION="${VERSION}" && cd frontend && $(YARN) build
 	touch -c $(FRONTEND_DIST)
 
+# Build the JS email-builder dist.
+$(FRONTEND_EMAIL_BUILDER_DIST): $(FRONTEND_EMAIL_BUILDER_DEPS)
+	export VUE_APP_VERSION="${VERSION}" && cd $(FRONTEND_EMAIL_BUILDER) && $(YARN) build
+	touch -c $(FRONTEND_EMAIL_BUILDER_DIST)
+
+# Copy the build assets to frontend.
+$(FRONTEND_EMAIL_BUILDER_DIST_FINAL): $(FRONTEND_EMAIL_BUILDER_DIST)
+	mkdir -p $(FRONTEND_EMAIL_BUILDER_DIST_FINAL)
+	cp -r $(FRONTEND_EMAIL_BUILDER_DIST)/* $(FRONTEND_EMAIL_BUILDER_DIST_FINAL)
+	touch -c $(FRONTEND_EMAIL_BUILDER_DIST_FINAL)
 
 .PHONY: build-frontend
-build-frontend: $(FRONTEND_DIST)
+build-frontend: $(FRONTEND_EMAIL_BUILDER_DIST_FINAL) $(FRONTEND_DIST)
+
+.PHONY: build-email-builder
+build-email-builder: $(FRONTEND_EMAIL_BUILDER_DIST_FINAL)
 
 # Run the JS frontend server in dev mode.
 .PHONY: run-frontend
-run-frontend:
+run-frontend: $(FRONTEND_EMAIL_BUILDER_DIST_FINAL)
 	export VUE_APP_VERSION="${VERSION}" && cd frontend && $(YARN) dev
 
 # Run Go tests.

cmd/campaigns.go

@@ -88,6 +88,7 @@ func (a *App) GetCampaigns(c echo.Context) error {
 	if noBody {
 		for i := range res {
 			res[i].Body = ""
+			res[i].BodySource.Valid = false
 		}
 	}
 
@@ -142,17 +143,31 @@ func (a *App) PreviewCampaign(c echo.Context) error {
 		return err
 	}
 
-	// Fetch the campaign body from the DB.
-	tplID, _ := strconv.Atoi(c.FormValue("template_id"))
+	var (
+		isPost      = c.Request().Method == http.MethodPost
+		contentType = c.FormValue("content_type")
+		tplID, _    = strconv.Atoi(c.FormValue("template_id"))
+	)
+	// For visual content, template ID for previewing is irrelevant.
+	if contentType == models.CampaignContentTypeVisual || tplID < 1 {
+		tplID = 0
+	}
+
+	// Get the campaign from the DB for previewing with the `template_body` field.
 	camp, err := a.core.GetCampaignForPreview(id, tplID)
 	if err != nil {
 		return err
 	}
 
 	// There's a body in the request to preview instead of the body in the DB.
-	if c.Request().Method == http.MethodPost {
-		camp.ContentType = c.FormValue("content_type")
+	if isPost {
+		camp.ContentType = contentType
 		camp.Body = c.FormValue("body")
+
+		// For visual campaigns, template body from the DB shouldn't be used.
+		if contentType == models.CampaignContentTypeVisual {
+			camp.TemplateBody = ""
+		}
 	}
 
 	// Use a dummy campaign ID to prevent views and clicks from {{ TrackView }}
@@ -172,13 +187,52 @@ func (a *App) PreviewCampaign(c echo.Context) error {
 			a.i18n.Ts("templates.errorRendering", "error", err.Error()))
 	}
 
+	// Plaintext headers for plain body.
 	if camp.ContentType == models.CampaignContentTypePlain {
 		return c.String(http.StatusOK, string(msg.Body()))
 	}
 
 	return c.HTML(http.StatusOK, string(msg.Body()))
 }
 
+// PreviewCampaignArchive renders the public campaign archives page.
+func (a *App) PreviewCampaignArchive(c echo.Context) error {
+	// Get the campaign ID.
+	id := getID(c)
+
+	// Check if the user has access to the campaign.
+	if err := a.checkCampaignPerm(auth.PermTypeGet, id, c); err != nil {
+		return err
+	}
+
+	// Fetch the campaign body from the DB.
+	tplID, _ := strconv.Atoi(c.FormValue("template_id"))
+	camp, err := a.core.GetCampaignForPreview(id, tplID)
+	if err != nil {
+		return err
+	}
+
+	camp.ArchiveMeta = json.RawMessage([]byte(c.FormValue("archive_meta")))
+
+	// "Compile" the campaign template with appropriate data.
+	res, err := a.compileArchiveCampaigns([]models.Campaign{camp})
+	if err != nil {
+		return c.Render(http.StatusInternalServerError, tplMessage,
+			makeMsgTpl(a.i18n.T("public.errorTitle"), "", a.i18n.Ts("public.errorFetchingCampaign")))
+	}
+
+	// Render the campaign body.
+	out := res[0].Campaign
+	msg, err := a.manager.NewCampaignMessage(out, res[0].Subscriber)
+	if err != nil {
+		a.log.Printf("error rendering campaign: %v", err)
+		return c.Render(http.StatusInternalServerError, tplMessage,
+			makeMsgTpl(a.i18n.T("public.errorTitle"), "", a.i18n.Ts("public.errorFetchingCampaign")))
+	}
+
+	return c.HTML(http.StatusOK, string(msg.Body()))
+}
+
 // CampaignContent handles campaign content (body) format conversions.
 func (a *App) CampaignContent(c echo.Context) error {
 	var camp campContentReq
@@ -214,9 +268,6 @@ func (a *App) CreateCampaign(c echo.Context) error {
 		o.Type = models.CampaignTypeRegular
 	}
 
-	if o.ContentType == "" {
-		o.ContentType = models.CampaignContentTypeRichtext
-	}
 	if o.Messenger == "" {
 		o.Messenger = "email"
 	}
@@ -228,7 +279,7 @@ func (a *App) CreateCampaign(c echo.Context) error {
 		o = c
 	}
 
-	if o.ArchiveTemplateID == 0 {
+	if o.ArchiveTemplateID.Valid && o.ArchiveTemplateID.Int != 0 {
 		o.ArchiveTemplateID = o.TemplateID
 	}
 
@@ -553,6 +604,16 @@ func (a *App) validateCampaignFields(c campReq) (campReq, error) {
 		return c, errors.New(a.i18n.T("campaigns.fieldInvalidSubject"))
 	}
 
+	// If no content-type is specified, default to richtext.
+	if c.ContentType != models.CampaignContentTypeRichtext &&
+		c.ContentType != models.CampaignContentTypeHTML &&
+		c.ContentType != models.CampaignContentTypePlain &&
+		c.ContentType != models.CampaignContentTypeVisual &&
+		c.ContentType != models.CampaignContentTypeMarkdown {
+		c.ContentType = models.CampaignContentTypeRichtext
+		c.BodySource.Valid = false
+	}
+
 	// If there's a "send_at" date, it should be in the future.
 	if c.SendAt.Valid {
 		if c.SendAt.Time.Before(time.Now()) {

cmd/handlers.go

@@ -150,6 +150,7 @@ func initHTTPHandlers(e *echo.Echo, a *App) {
 		g.GET("/api/campaigns/:id", pm(hasID(a.GetCampaign), "campaigns:get_all", "campaigns:get"))
 		g.GET("/api/campaigns/analytics/:type", pm(a.GetCampaignViewAnalytics, "campaigns:get_analytics"))
 		g.GET("/api/campaigns/:id/preview", pm(hasID(a.PreviewCampaign), "campaigns:get_all", "campaigns:get"))
+		g.POST("/api/campaigns/:id/preview/archive", pm(hasID(a.PreviewCampaignArchive), "campaigns:get_all", "campaigns:get"))
 		g.POST("/api/campaigns/:id/preview", pm(hasID(a.PreviewCampaign), "campaigns:get_all", "campaigns:get"))
 		g.POST("/api/campaigns/:id/content", pm(hasID(a.CampaignContent), "campaigns:manage_all", "campaigns:manage"))
 		g.POST("/api/campaigns/:id/text", pm(hasID(a.PreviewCampaign), "campaigns:get"))

cmd/init.go

@@ -316,7 +316,7 @@ func initDB() *sqlx.DB {
 	db.SetMaxIdleConns(c.MaxIdle)
 	db.SetConnMaxLifetime(c.MaxLifetime)
 
-	return db
+	return db.Unsafe()
 }
 
 // readQueries reads named SQL queries from the SQL queries file into a query map.
@@ -358,7 +358,7 @@ func prepareQueries(qMap goyesql.Queries, db *sqlx.DB, ko *koanf.Koanf) *models.
 
 	// Scan and prepare all queries.
 	var q models.Queries
-	if err := goyesqlx.ScanToStruct(&q, qMap, db.Unsafe()); err != nil {
+	if err := goyesqlx.ScanToStruct(&q, qMap, db); err != nil {
 		lo.Fatalf("error preparing SQL queries: %v", err)
 	}
 

cmd/install.go

@@ -193,7 +193,7 @@ func installTemplates(q *models.Queries) (int, int) {
 	}
 
 	var campTplID int
-	if err := q.CreateTemplate.Get(&campTplID, "Default campaign template", models.TemplateTypeCampaign, "", campTpl.ReadBytes()); err != nil {
+	if err := q.CreateTemplate.Get(&campTplID, "Default campaign template", models.TemplateTypeCampaign, "", campTpl.ReadBytes(), nil); err != nil {
 		lo.Fatalf("error creating default campaign template: %v", err)
 	}
 	if _, err := q.SetDefaultTemplate.Exec(campTplID); err != nil {
@@ -207,7 +207,7 @@ func installTemplates(q *models.Queries) (int, int) {
 	}
 
 	var archiveTplID int
-	if err := q.CreateTemplate.Get(&archiveTplID, "Default archive template", models.TemplateTypeCampaign, "", archiveTpl.ReadBytes()); err != nil {
+	if err := q.CreateTemplate.Get(&archiveTplID, "Default archive template", models.TemplateTypeCampaign, "", archiveTpl.ReadBytes(), nil); err != nil {
 		lo.Fatalf("error creating default campaign template: %v", err)
 	}
 
@@ -217,10 +217,24 @@ func installTemplates(q *models.Queries) (int, int) {
 		lo.Fatalf("error reading default e-mail template: %v", err)
 	}
 
-	if _, err := q.CreateTemplate.Exec("Sample transactional template", models.TemplateTypeTx, "Welcome {{ .Subscriber.Name }}", txTpl.ReadBytes()); err != nil {
+	if _, err := q.CreateTemplate.Exec("Sample transactional template", models.TemplateTypeTx, "Welcome {{ .Subscriber.Name }}", txTpl.ReadBytes(), nil); err != nil {
 		lo.Fatalf("error creating sample transactional template: %v", err)
 	}
 
+	// Sample visual campaign template.
+	visualTpl, err := fs.Get("/static/email-templates/default-visual.tpl")
+	if err != nil {
+		lo.Fatalf("error reading default visual template: %v", err)
+	}
+	visualSrc, err := fs.Get("/static/email-templates/default-visual.json")
+	if err != nil {
+		lo.Fatalf("error reading default visual template json: %v", err)
+	}
+
+	if _, err := q.CreateTemplate.Exec("Sample visual template", models.TemplateTypeCampaignVisual, "", visualTpl.ReadBytes(), visualSrc.ReadBytes()); err != nil {
+		lo.Fatalf("error creating default campaign template: %v", err)
+	}
+
 	return campTplID, archiveTplID
 }
 
@@ -252,6 +266,7 @@ func installCampaign(campTplID, archiveTplID int, q *models.Queries) {
 		archiveTplID,
 		`{"name": "Subscriber"}`,
 		nil,
+		nil,
 	); err != nil {
 		lo.Fatalf("error creating sample campaign: %v", err)
 	}

cmd/templates.go

@@ -117,7 +117,7 @@ func (a *App) CreateTemplate(c echo.Context) error {
 	// Subject is only relevant for fixed tx templates. For campaigns,
 	// the subject changes per campaign and is on models.Campaign.
 	var funcs template.FuncMap
-	if o.Type == models.TemplateTypeCampaign {
+	if o.Type == models.TemplateTypeCampaign || o.Type == models.TemplateTypeCampaignVisual {
 		o.Subject = ""
 		funcs = a.manager.TemplateFuncs(nil)
 	} else {
@@ -130,7 +130,7 @@ func (a *App) CreateTemplate(c echo.Context) error {
 	}
 
 	// Create the template the in the DB.
-	out, err := a.core.CreateTemplate(o.Name, o.Type, o.Subject, []byte(o.Body))
+	out, err := a.core.CreateTemplate(o.Name, o.Type, o.Subject, []byte(o.Body), o.BodySource)
 	if err != nil {
 		return err
 	}
@@ -171,7 +171,7 @@ func (a *App) UpdateTemplate(c echo.Context) error {
 
 	// Update the template in the DB.
 	id := getID(c)
-	out, err := a.core.UpdateTemplate(id, o.Name, o.Subject, []byte(o.Body))
+	out, err := a.core.UpdateTemplate(id, o.Name, o.Subject, []byte(o.Body), o.BodySource)
 	if err != nil {
 		return err
 	}
@@ -232,7 +232,7 @@ func (a *App) validateTemplate(o models.Template) error {
 // previewTemplate renders the HTML preview of a template.
 func (a *App) previewTemplate(tpl models.Template) ([]byte, error) {
 	var out []byte
-	if tpl.Type == models.TemplateTypeCampaign {
+	if tpl.Type == models.TemplateTypeCampaign || tpl.Type == models.TemplateTypeCampaignVisual {
 		camp := models.Campaign{
 			UUID:         dummyUUID,
 			Name:         a.i18n.T("templates.dummyName"),

cmd/upgrade.go

@@ -40,7 +40,7 @@ var migList = []migFunc{
 	{"v3.0.0", migrations.V3_0_0},
 	{"v4.0.0", migrations.V4_0_0},
 	{"v4.1.0", migrations.V4_1_0},
-	{"v5.0.0", migrations.V5_0_0},
+	{"v5.0.0-rc.1", migrations.V5_0_0_rc1},
 }
 
 // upgrade upgrades the database to the current version by running SQL migration files

frontend/.eslintrc.js

@@ -28,4 +28,5 @@ module.exports = {
       comments: 200,
     }],
   },
+  ignorePatterns: ['src/email-builder.js'],
 };