feat(dpkg): license parser added

analyzer/analyzer.go

@@ -147,6 +147,10 @@ func (r *AnalysisResult) Sort() {
 		})
 	}
 
+	sort.Slice(r.CustomResources, func(i, j int) bool {
+		return r.CustomResources[i].FilePath < r.CustomResources[j].FilePath
+	})
+
 	for _, files := range r.Files {
 		sort.Slice(files, func(i, j int) bool {
 			return files[i].Path < files[j].Path

analyzer/analyzer_test.go

@@ -499,19 +499,23 @@ func TestAnalyzer_AnalyzerVersions(t *testing.T) {
 			name:     "happy path",
 			disabled: []analyzer.Type{},
 			want: map[string]int{
-				"alpine":   1,
-				"apk-repo": 1,
-				"apk":      1,
-				"bundler":  1,
-				"ubuntu":   1,
+				"alpine":       1,
+				"apk-repo":     1,
+				"apk":          1,
+				"bundler":      1,
+				"ubuntu":       1,
+				"dpkg":         2,
+				"dpkg-licence": 1,
 			},
 		},
 		{
 			name:     "disable analyzers",
 			disabled: []analyzer.Type{analyzer.TypeAlpine, analyzer.TypeApkRepo, analyzer.TypeUbuntu},
 			want: map[string]int{
-				"apk":     1,
-				"bundler": 1,
+				"apk":          1,
+				"bundler":      1,
+				"dpkg":         2,
+				"dpkg-licence": 1,
 			},
 		},
 	}

analyzer/const.go

@@ -91,6 +91,11 @@ const (
 	// =======
 	TypeRedHatContentManifestType = "redhat-content-manifest"
 	TypeRedHatDockerfileType      = "redhat-dockerfile"
+
+	// =======
+	// Debian
+	// =======
+	TypeDpkgLicence = "dpkg-licence"
 )
 
 var (

analyzer/pkg/dpkg/copyright.go

@@ -0,0 +1,111 @@
+package dpkg
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/aquasecurity/fanal/analyzer"
+	"github.com/aquasecurity/fanal/types"
+	classifier "github.com/google/licenseclassifier/v2/assets"
+	"golang.org/x/exp/slices"
+	"golang.org/x/xerrors"
+)
+
+func init() {
+	analyzer.RegisterAnalyzer(&dpkgLicencesAnalyzer{})
+}
+
+const LicenseAdder = "dpkg-license-adder"
+
+var (
+	dpkgLicencesAnalyzerVersion = 1
+
+	cl, _                        = classifier.DefaultClassifier()
+	copyrightFileRegexp          = regexp.MustCompile(`^usr/share/doc/([0-9A-Za-z_.-]+)/copyright$`)
+	commonLicenseReferenceRegexp = regexp.MustCompile(`/?usr/share/common-licenses/([0-9A-Za-z_.+-]+[0-9A-Za-z+])`)
+)
+
+type dpkgLicencesAnalyzer struct{}
+
+func (a dpkgLicencesAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) {
+	scanner := bufio.NewScanner(input.Content)
+	return parseCopyrightFile(input, scanner)
+}
+
+// parseCopyrightFile parses /usr/share/doc/*/copyright files
+func parseCopyrightFile(input analyzer.AnalysisInput, scanner *bufio.Scanner) (*analyzer.AnalysisResult, error) {
+	var licenses []string
+	buf, err := ioutil.ReadAll(input.Content) // save stream to buffer for use at github.com/google/licenseclassifier
+	if err != nil {
+		return nil, xerrors.Errorf("unable to read content from %q: %w", input.FilePath, err)
+	}
+	if _, err := input.Content.Seek(0, io.SeekStart); err != nil { // rewind the reader to the beginning of the stream after saving
+		return nil, xerrors.Errorf("unable to rewind reader for %q file: %w", input.FilePath, err)
+	}
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		// 'License: *' pattern is used
+		if strings.HasPrefix(line, "License:") {
+			l := strings.TrimSpace(line[8:])
+			if !slices.Contains(licenses, l) {
+				licenses = append(licenses, l)
+			}
+		} else {
+			// Common license pattern is used
+			license := commonLicenseReferenceRegexp.FindStringSubmatch(line)
+			if len(license) == 2 && !slices.Contains(licenses, license[1]) {
+				licenses = append(licenses, license[1])
+			}
+		}
+	}
+
+	// Use 'github.com/google/licenseclassifier' for find licenses
+	result := cl.Match(buf)
+	for _, match := range result.Matches {
+		if !slices.Contains(licenses, match.Name) {
+			licenses = append(licenses, match.Name)
+		}
+	}
+
+	licensesStr := strings.Join(licenses, ", ")
+	if licensesStr == "" {
+		licensesStr = "Unknown"
+	}
+
+	return &analyzer.AnalysisResult{
+		CustomResources: []types.CustomResource{
+			{
+				Type:     LicenseAdder,
+				FilePath: getPkgNameFromLicenseFilePath(input.FilePath),
+				Data:     licensesStr,
+			},
+		},
+	}, nil
+}
+
+func (a dpkgLicencesAnalyzer) Required(filePath string, _ os.FileInfo) bool {
+	return copyrightFileRegexp.MatchString(filePath)
+}
+
+func (a dpkgLicencesAnalyzer) Type() analyzer.Type {
+	return analyzer.TypeDpkgLicence
+}
+
+func (a dpkgLicencesAnalyzer) Version() int {
+	return dpkgLicencesAnalyzerVersion
+}
+
+func getPkgNameFromLicenseFilePath(filePath string) string {
+	pkgName := copyrightFileRegexp.FindStringSubmatch(filePath)
+	if len(pkgName) == 2 {
+		return pkgName[1]
+	}
+	return ""
+}

analyzer/pkg/dpkg/copyright_test.go

@@ -0,0 +1,150 @@
+package dpkg
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/aquasecurity/fanal/analyzer"
+	"github.com/aquasecurity/fanal/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDpkgLicencesAnalyzer_Analyze(t *testing.T) {
+	tests := []struct {
+		name              string
+		copyrightFilePath string
+		wantLicense       *analyzer.AnalysisResult
+	}{
+		{
+			name:              "happy path. There are 'License:' pattern and licenseclassifier",
+			copyrightFilePath: "testdata/copyrightFiles/usr/share/doc/zlib1g/copyright",
+			wantLicense: &analyzer.AnalysisResult{
+				CustomResources: []types.CustomResource{
+					{
+						Type:     LicenseAdder,
+						FilePath: "zlib1g",
+						Data:     "Zlib",
+					},
+				},
+			},
+		},
+		{
+			name:              "happy path. There is Common license",
+			copyrightFilePath: "testdata/copyrightFiles/usr/share/doc/adduser/copyright",
+			wantLicense: &analyzer.AnalysisResult{
+				CustomResources: []types.CustomResource{
+					{
+						Type:     LicenseAdder,
+						FilePath: "adduser",
+						Data:     "GPL-2, GPL-2.0",
+					},
+				},
+			},
+		},
+		{
+			name:              "happy path. There are Common license, 'License:' pattern and licenseclassifier",
+			copyrightFilePath: "testdata/copyrightFiles/usr/share/doc/apt/copyright",
+			wantLicense: &analyzer.AnalysisResult{
+				CustomResources: []types.CustomResource{
+					{
+						Type:     LicenseAdder,
+						FilePath: "apt",
+						Data:     "GPLv2+, GPL-2, GPL-2.0",
+					},
+				},
+			},
+		},
+		{
+			name:              "happy path. Licenses not found",
+			copyrightFilePath: "testdata/copyrightFiles/usr/share/doc/tzdata/copyright",
+			wantLicense: &analyzer.AnalysisResult{
+				CustomResources: []types.CustomResource{
+					{
+						Type:     LicenseAdder,
+						FilePath: "tzdata",
+						Data:     "Unknown",
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			f, err := os.Open(test.copyrightFilePath)
+			if err != nil {
+				t.Error("unable to read test file")
+			}
+
+			input := analyzer.AnalysisInput{
+				Content:  f,
+				FilePath: strings.TrimPrefix(test.copyrightFilePath, "testdata/copyrightFiles/"),
+			}
+			a := dpkgLicencesAnalyzer{}
+
+			license, _ := a.Analyze(context.Background(), input)
+			assert.Equal(t, test.wantLicense, license)
+		})
+	}
+}
+
+func TestDpkgLicencesAnalyzer_Required(t *testing.T) {
+	tests := []struct {
+		name     string
+		filePath string
+		want     bool
+	}{
+		{
+			name:     "happy path",
+			filePath: "usr/share/doc/eject/copyright",
+			want:     true,
+		},
+		{
+			name:     "sad path. Wrong path",
+			filePath: "/usr/share/doc/library/eject/copyright",
+			want:     false,
+		},
+		{
+			name:     "sad path. Wrong prefix",
+			filePath: "/usr/share/doc/eject/copyright/file",
+			want:     false,
+		},
+		{
+			name:     "sad path. Wrong suffix",
+			filePath: "/usr/share/doc/eject/copyright",
+			want:     false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			a := dpkgLicencesAnalyzer{}
+			assert.Equal(t, test.want, a.Required(test.filePath, nil))
+		})
+	}
+}
+
+func TestDpkgAnalyzer_getPkgNameFromLicenseFilePath(t *testing.T) {
+	tests := []struct {
+		name     string
+		filePath string
+		wantPkg  string
+	}{
+		{
+			name:     "happy path",
+			filePath: "usr/share/doc/eject/copyright",
+			wantPkg:  "eject",
+		},
+		{
+			name:     "sad path",
+			filePath: "usr/share/doc/library/eject/copyright",
+			wantPkg:  "",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.wantPkg, getPkgNameFromLicenseFilePath(test.filePath))
+		})
+	}
+}

analyzer/pkg/dpkg/testdata/copyrightFiles/usr/share/doc/adduser/copyright

@@ -0,0 +1,47 @@
+This package was first put together by Ian Murdock
+<imurdock@debian.org> and was maintained by Steve Phillips
+<sjp@cvfn.org> from sources written for the Debian Project by Ian
+Murdock, Ted Hajek <tedhajek@boombox.micro.umn.edu>, and Sven Rudolph
+<sr1@inf.tu-dresden.de>.
+
+Since Nov 27 1996, it was maintained by Guy Maor <maor@debian.org>.  He
+rewrote most of it.
+
+Since May 20 2000, it is maintained by Roland Bauerschmidt
+<rb@debian.org>.
+
+Since March 24 2004, it is maintained by Roland Bauerschmidt
+<rb@debian.org>, and co-maintained by Marc Haber
+<mh+debian-packages@zugschlus.de>
+
+Since 23 Oct 2005, it has been maintained by Joerg Hoh <joerg@joerghoh.de> 
+
+Since June 2006, it has been maintained by Stephen Gran <sgran@debian.org>
+
+deluser is Copyright (C) 2000 Roland Bauerschmidt <rb@debian.org>
+and based on the source code of adduser.
+
+adduser is Copyright (C) 1997, 1998, 1999 Guy Maor <maor@debian.org>.
+adduser is Copyright (C) 1995 Ted Hajek <tedhajek@boombox.micro.umn.edu>
+with portions Copyright (C) 1994 Debian Association, Inc.
+
+The examples directory has been contributed by John Zaitseff, and is
+GPL V2 as well.
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the
+    Free Software Foundation, Inc.,
+    51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+On Debian GNU/Linux systems, the complete text of the GNU General
+Public License can be found in `/usr/share/common-licenses/GPL-2'.

analyzer/pkg/dpkg/testdata/copyrightFiles/usr/share/doc/apt/copyright

@@ -0,0 +1,22 @@
+Apt is copyright 1997, 1998, 1999 Jason Gunthorpe and others.
+Apt is currently developed by APT Development Team <deity@lists.debian.org>.
+
+License: GPLv2+
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+See /usr/share/common-licenses/GPL-2, or
+<http://www.gnu.org/copyleft/gpl.txt> for the terms of the latest version
+of the GNU General Public License.

analyzer/pkg/dpkg/testdata/copyrightFiles/usr/share/doc/tzdata/copyright

@@ -0,0 +1,10 @@
+This is the Debian prepackaged version of the Time Zone and Daylight 
+Saving Time Data. 
+
+It was downloaded from http://www.iana.org/time-zones
+
+Upstream Author: The Internet Assigned Numbers Authority (IANA)
+Commentary should be addressed to tz@iana.org
+
+Copyright: This database is in the public domain.
+