diff --git a/Makefile b/Makefile index 805d259b4bb..9cbcaa841a7 100644 --- a/Makefile +++ b/Makefile @@ -374,7 +374,7 @@ amazon-linux-sources.tgz: amazon-linux-rpm-integrated: .amazon-linux-rpm-integrated-done -.generic-rpm-integrated-done: +.generic-rpm-integrated-done: get-cni-sources ./scripts/update-version.sh cp packaging/generic-rpm-integrated/amazon-ecs-init.spec amazon-ecs-init.spec cp packaging/generic-rpm-integrated/ecs.service ecs.service @@ -388,7 +388,40 @@ amazon-linux-rpm-integrated: .amazon-linux-rpm-integrated-done generic-rpm-integrated: .generic-rpm-integrated-done -.generic-rpm-done: +VERSION = $(shell cat ecs-init/ECSVERSION) + +.generic-deb-integrated-done: get-cni-sources + mkdir -p BUILDROOT + ./scripts/update-version.sh + tar -czf ./amazon-ecs-init_${VERSION}.orig.tar.gz ecs-init scripts README.md + cp -r packaging/generic-deb-integrated/debian ecs-init scripts misc agent agent-container amazon-ecs-cni-plugins amazon-vpc-cni-plugins README.md VERSION BUILDROOT + cd BUILDROOT && debuild -uc -us --lintian-opts --suppress-tags bad-distribution-in-changes-file,file-in-unusual-dir + touch .generic-deb-integrated-done + +generic-deb-integrated: .generic-deb-integrated-done + +ARCH:=$(shell uname -m) +ifeq (${ARCH},x86_64) + AGENT_FILENAME=ecs-agent-v${VERSION}.tar +else ifeq (${ARCH},aarch64) + AGENT_FILENAME=ecs-agent-arm64-v${VERSION}.tar +# osx M1 instances +else ifeq (${ARCH},arm64) + AGENT_FILENAME=ecs-agent-arm64-v${VERSION}.tar +endif + +BUILDROOT/ecs-agent.tar: + mkdir -p BUILDROOT + curl -o BUILDROOT/ecs-agent.tar https://s3.amazonaws.com/amazon-ecs-agent/${AGENT_FILENAME} + +${AGENT_FILENAME}: BUILDROOT/ecs-agent.tar + cp BUILDROOT/ecs-agent.tar ${AGENT_FILENAME} + +rpm-in-docker: ${AGENT_FILENAME} + docker build -t "amazon/amazon-ecs-init:build" -f "scripts/dockerfiles/build.dockerfile" . + docker run -u "$(shell id -u)" --tmpfs /.cache -v "$(shell pwd):/workspace/amazon-ecs-init" "amazon/amazon-ecs-init:build" + +.generic-rpm-done: ${AGENT_FILENAME} ./scripts/update-version.sh cp packaging/generic-rpm/amazon-ecs-init.spec amazon-ecs-init.spec cp packaging/generic-rpm/ecs.service ecs.service @@ -409,13 +442,10 @@ generic-rpm: .generic-rpm-done cd BUILDROOT && debuild -uc -us --lintian-opts --suppress-tags bad-distribution-in-changes-file,file-in-unusual-dir touch .deb-done +.PHONY: deb deb: .deb-done clean: - # ensure docker is running and we can talk to it, abort if not: - docker ps > /dev/null - -docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild:make" - -docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild-windows:make" rm -f misc/certs/host-certs.crt &> /dev/null rm -rf misc/pause-container/image/ rm -rf misc/pause-container/rootfs/ @@ -423,12 +453,6 @@ clean: rm -rf out/ rm -rf rootfs/ -$(MAKE) -C $(ECS_CNI_REPOSITORY_SRC_DIR) clean - -$(MAKE) -C misc/netkitten $(MFLAGS) clean - -$(MAKE) -C misc/volumes-test $(MFLAGS) clean - -$(MAKE) -C misc/exec-command-agent-test $(MFLAGS) clean - -$(MAKE) -C misc/gremlin $(MFLAGS) clean - -$(MAKE) -C misc/image-cleanup-test-images $(MFLAGS) clean - -$(MAKE) -C misc/container-health $(MFLAGS) clean -rm -f .get-deps-stamp -rm -f .builder-image-stamp -rm -f .out-stamp @@ -460,3 +484,16 @@ clean: -rm -f .amazon-linux-rpm-integrated-done -rm -f .generic-rpm-integrated-done -rm -f amazon-ecs-volume-plugin + +clean-all: clean + # for our dockerfree builds, we likely don't have docker + # ensure docker is running and we can talk to it, abort if not: + docker ps > /dev/null + -docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild:make" + -docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild-windows:make" + -$(MAKE) -C misc/netkitten $(MFLAGS) clean + -$(MAKE) -C misc/volumes-test $(MFLAGS) clean + -$(MAKE) -C misc/exec-command-agent-test $(MFLAGS) clean + -$(MAKE) -C misc/gremlin $(MFLAGS) clean + -$(MAKE) -C misc/image-cleanup-test-images $(MFLAGS) clean + -$(MAKE) -C misc/container-health $(MFLAGS) clean diff --git a/build-infrastructure/codebuild-devbuild-stack.yml b/build-infrastructure/codebuild-devbuild-stack.yml index 323c7bce11f..00069dd83a8 100644 --- a/build-infrastructure/codebuild-devbuild-stack.yml +++ b/build-infrastructure/codebuild-devbuild-stack.yml @@ -19,6 +19,60 @@ Parameters: Description: Base name of CodeBuild projects - will start different CodeBuild projects with different suffixes (-amd, -arm) for different artifacts Resources: + UbuntuAmdProject: + Type: 'AWS::CodeBuild::Project' + Properties: + Artifacts: + Location: !Ref BuildBucketName + NamespaceType: NONE + OverrideArtifactName: true + Packaging: NONE + Path: development + Type: S3 + BadgeEnabled: false + ConcurrentBuildLimit: 10 + Description: A CodeBuild project to build artifacts (AMD/x86_64). Builds are triggered by PR creation and updates, and artifacts are saved in S3 + Environment: + ComputeType: BUILD_GENERAL1_SMALL + Image: 'aws/codebuild/standard:5.0' + ImagePullCredentialsType: CODEBUILD + PrivilegedMode: false + Type: LINUX_CONTAINER + Name: !Sub '${BuildProjectName}-ubuntu-amd' + QueuedTimeoutInMinutes: 60 + ServiceRole: !Ref ServiceRoleUbuntuAmd + Source: + BuildSpec: buildspecs/pr-build-ubuntu.yml + Location: !Ref GithubFullRepoName + Type: GITHUB + TimeoutInMinutes: 60 + Triggers: + BuildType: BUILD + # Config list of developers allowlisted to create builds when creating PRs to GithubBranchName + # This allow list can be modified using aws-cli or aws-sdk + # CodeBuild also supports pattern matches using regex, but this is not useful for listing different Github IDs + # so they have to be listed separately + FilterGroups: + - - Type: EVENT + Pattern: 'PULL_REQUEST_CREATED,PULL_REQUEST_UPDATED,PULL_REQUEST_REOPENED' + - Type: BASE_REF + Pattern: !Sub '^${GithubBranchName}$' + - Type: ACTOR_ACCOUNT_ID + Pattern: '5080306' # prateekchaudhry + - - Type: EVENT + Pattern: 'PULL_REQUEST_CREATED,PULL_REQUEST_UPDATED,PULL_REQUEST_REOPENED' + - Type: BASE_REF + Pattern: !Sub '^${GithubBranchName}$' + - Type: ACTOR_ACCOUNT_ID + Pattern: '4751028' # fierlion + - - Type: EVENT + Pattern: 'PULL_REQUEST_CREATED,PULL_REQUEST_UPDATED,PULL_REQUEST_REOPENED' + - Type: BASE_REF + Pattern: !Sub '^${GithubBranchName}$' + - Type: ACTOR_ACCOUNT_ID + Pattern: '3102848' # YashdalfTheGray + Webhook: true + Visibility: PRIVATE ArmProject: Type: 'AWS::CodeBuild::Project' Properties: @@ -180,6 +234,59 @@ Resources: - 's3:GetBucketLocation' PolicyName: !Sub '${AWS::StackName}-ServicePolicyAmd' RoleName: !Sub '${AWS::StackName}-ServiceRoleAmd' + ServiceRoleUbuntuAmd: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: 'sts:AssumeRole' + Description: Service role, allow access to CW and S3 + Path: / + Policies: + - PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Resource: + - !Join + - ':' + - - 'arn:aws:logs' + - !Ref 'AWS::Region' + - !Ref 'AWS::AccountId' + - !Sub 'log-group:/aws/codebuild/${BuildProjectName}-ubuntu-amd' + - !Join + - ':' + - - 'arn:aws:logs' + - !Ref 'AWS::Region' + - !Ref 'AWS::AccountId' + - !Sub 'log-group:/aws/codebuild/${BuildProjectName}-ubuntu-amd:*' + Action: + - 'logs:CreateLogGroup' + - 'logs:CreateLogStream' + - 'logs:PutLogEvents' + - Effect: Allow + Resource: + - 'arn:aws:s3:::codepipeline-us-west-2-*' + Action: + - 's3:PutObject' + - 's3:GetObject' + - 's3:GetObjectVersion' + - 's3:GetBucketAcl' + - 's3:GetBucketLocation' + - Effect: Allow + Resource: + - !Sub '${BuildBucketArn}/*' + Action: + - 's3:GetObject' + - 's3:PutObject' + - 's3:GetBucketAcl' + - 's3:GetBucketLocation' + PolicyName: !Sub '${AWS::StackName}-ServicePolicyUbuntuAmd' + RoleName: !Sub '${AWS::StackName}-ServiceRoleUbuntuAmd' ServiceRoleArm: Type: 'AWS::IAM::Role' Properties: @@ -232,4 +339,4 @@ Resources: - 's3:GetBucketAcl' - 's3:GetBucketLocation' PolicyName: !Sub '${AWS::StackName}-ServicePolicyArm' - RoleName: !Sub '${AWS::StackName}-ServiceRoleArm' + RoleName: !Sub '${AWS::StackName}-ServiceRoleArm' \ No newline at end of file diff --git a/build-infrastructure/release-pipeline-stack.yml b/build-infrastructure/release-pipeline-stack.yml index 13c62302172..d84ed33c703 100644 --- a/build-infrastructure/release-pipeline-stack.yml +++ b/build-infrastructure/release-pipeline-stack.yml @@ -19,6 +19,10 @@ Parameters: Type: String Description: The name of the ARM build project Default: arm-artifact-build + UbuntuAmdBuildCodeBuildProjectName: + Type: String + Description: The name of the Ubuntu AMD build project (builds .deb ) + Default: ubuntu-amd-artifact-build SigningCodeBuildProjectName: Type: String Description: The name of the signing project @@ -65,16 +69,16 @@ Parameters: Description: The ARN of the bucket where things land at the end, this is assumed to already exist ReleaseArtifactsBucketS3Uri: Type: String - Description: The URI of the bucket where things land at the end, this is assumed to already exist + Description: The URI of the bucket where things land at the end, this is assumed to already exist (e.g. s3://artifacts) CodeStarConnectionArn: Type: String Description: The ARN of the connection to use to connect to GitHub GithubFullRepoName: Type: String - Description: The name of the repository that we want to use + Description: The name of the repository that we want to use (e.g. aws/amazon-ecs-agent) GithubBranchName: Type: String - Description: The name of the branch to use to build + Description: The name of the branch to use to build (e.g. mainline, dev) SecretKeyArn: Type: String Description: The ARN of the secret key @@ -189,7 +193,7 @@ Resources: Service: codebuild.amazonaws.com Action: sts:AssumeRole Policies: - - PolicyName: codebuild-build-base-policy + - PolicyName: amd-codebuild-build-base-policy PolicyDocument: Version: 2012-10-17 Statement: @@ -229,13 +233,65 @@ Resources: - codebuild:BatchPutTestCases - codebuild:BatchPutCodeCoverages + UbuntuAmdBuildCodeBuildProjectServiceRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Sub 'ubuntu-amd-build-codebuild-project-service-role-${AWS::Region}' + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: ubuntu-codebuild-build-base-policy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Sid: CloudWatchLogsAccess + Effect: Allow + Resource: + - !GetAtt CodeBuildLogGroup.Arn + - !Sub '${CodeBuildLogGroup.Arn}:*' + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - Sid: ArtifactBucketAccess + Effect: Allow + Resource: + - !Sub 'arn:aws:s3:::codepipeline-${AWS::Region}-*' + Action: + - s3:PutObject + - s3:GetObject + - s3:GetObjectVersion + - s3:GetBucketAcl + - s3:GetBucketLocation + - Sid: CodeBuildCodeStarConnectionAccess + Effect: Allow + Resource: + - !Ref CodeStarConnectionArn + Action: + - codestar-connections:UseConnection + - Sid: CodeBuildCreateReportAccess + Effect: Allow + Resource: + - !Sub 'arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:report-group/${UbuntuAmdBuildCodeBuildProjectName}-*' + Action: + - codebuild:CreateReportGroup + - codebuild:CreateReport + - codebuild:UpdateReport + - codebuild:BatchPutTestCases + - codebuild:BatchPutCodeCoverages + AmdBuildCodeBuildProject: Type: AWS::CodeBuild::Project Properties: Artifacts: Type: CODEPIPELINE ConcurrentBuildLimit: 10 - Description: CodeBuild project to build the ECS Agent Docker image tarball + Description: CodeBuild project to build the ECS Agent Docker image tarball and RPM Environment: ComputeType: BUILD_GENERAL1_SMALL Image: aws/codebuild/amazonlinux2-x86_64-standard:3.0 @@ -255,6 +311,32 @@ Resources: Type: CODEPIPELINE TimeoutInMinutes: 60 + UbuntuAmdBuildCodeBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Artifacts: + Type: CODEPIPELINE + ConcurrentBuildLimit: 10 + Description: CodeBuild project running an Ubuntu docker image to build the ECS Agent Docker deb packages + Environment: + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/standard:5.0 + ImagePullCredentialsType: CODEBUILD + PrivilegedMode: true + Type: LINUX_CONTAINER + LogsConfig: + CloudWatchLogs: + GroupName: !Ref CodeBuildLogGroupName + Status: ENABLED + StreamName: !Ref UbuntuAmdBuildCodeBuildProjectName + Name: !Ref UbuntuAmdBuildCodeBuildProjectName + QueuedTimeoutInMinutes: 60 + ServiceRole: !Ref UbuntuAmdBuildCodeBuildProjectServiceRole + Source: + BuildSpec: buildspecs/merge-build-ubuntu.yml + Type: CODEPIPELINE + TimeoutInMinutes: 60 + ArmBuildCodeBuildProjectServiceRole: Type: AWS::IAM::Role Properties: @@ -267,7 +349,7 @@ Resources: Service: codebuild.amazonaws.com Action: sts:AssumeRole Policies: - - PolicyName: codebuild-build-base-policy + - PolicyName: arm-codebuild-build-base-policy PolicyDocument: Version: 2012-10-17 Statement: @@ -700,6 +782,21 @@ Resources: - Name: AmdBuildArtifact RunOrder: 1 Namespace: AmdBuildVariables + - Name: MakeUbuntuAmd + InputArtifacts: + - Name: SourceArtifact + ActionTypeId: + Category: Build + Owner: AWS + Version: '1' + Provider: CodeBuild + Configuration: + ProjectName: !Ref UbuntuAmdBuildCodeBuildProject + EnvironmentVariables: '[{"name":"GIT_COMMIT_SHA","value":"#{SourceVariables.CommitId}","type":"PLAINTEXT"}]' + OutputArtifacts: + - Name: UbuntuAmdBuildArtifact + RunOrder: 1 + Namespace: UbuntuAmdBuildVariables - Name: MakeArm InputArtifacts: - Name: SourceArtifact @@ -720,6 +817,7 @@ Resources: - Name: GPG InputArtifacts: - Name: Buildspecs + - Name: UbuntuAmdBuildArtifact - Name: AmdBuildArtifact - Name: ArmBuildArtifact ActionTypeId: @@ -730,7 +828,14 @@ Resources: Configuration: ProjectName: !Ref SigningCodeBuildProject PrimarySource: Buildspecs - EnvironmentVariables: '[{"name":"GIT_COMMIT_SHA","value":"#{SourceVariables.CommitId}","type":"PLAINTEXT"},{"name":"ECS_AGENT_AMD_TAR","value":"#{AmdBuildVariables.ECS_AGENT_TAR}","type":"PLAINTEXT"},{"name":"ECS_AGENT_ARM_TAR","value":"#{ArmBuildVariables.ECS_AGENT_TAR}","type":"PLAINTEXT"}]' + # This project takes the following names as env vars + # - Commit sha + # - AMD tar + # - AMD rpm + # - Ubuntu AMD deb + # - ARM tar + # - ARM rpm + EnvironmentVariables: '[{"name":"GIT_COMMIT_SHA","value":"#{SourceVariables.CommitId}","type":"PLAINTEXT"},{"name":"ECS_AGENT_AMD_TAR","value":"#{AmdBuildVariables.ECS_AGENT_TAR}","type":"PLAINTEXT"},{"name":"ECS_AGENT_AMD_RPM","value":"#{AmdBuildVariables.ECS_AGENT_RPM}","type":"PLAINTEXT"},{"name":"ECS_AGENT_UBUNTU_AMD_DEB","value":"#{UbuntuAmdBuildVariables.ECS_AGENT_DEB}", "type":"PLAINTEXT"},{"name":"ECS_AGENT_ARM_TAR","value":"#{ArmBuildVariables.ECS_AGENT_TAR}","type":"PLAINTEXT"},{"name":"ECS_AGENT_ARM_RPM","value":"#{ArmBuildVariables.ECS_AGENT_RPM}","type":"PLAINTEXT"}]' OutputArtifacts: - Name: SignedArtifact RunOrder: 1 diff --git a/buildspecs/merge-build-ubuntu.yml b/buildspecs/merge-build-ubuntu.yml new file mode 100644 index 00000000000..e5704353f8b --- /dev/null +++ b/buildspecs/merge-build-ubuntu.yml @@ -0,0 +1,60 @@ +version: 0.2 + +env: + git-credential-helper: yes + exported-variables: + - CODEBUILD_BUILD_ID + - ECS_AGENT_DEB + - ECS_AGENT_DEB_TAR + +phases: + install: + commands: + - architecture="" + # Same buildspec for different architectures - detect the architecture here and rename the artifacts accordingly + - | + case $(uname -m) in + x86_64) + architecture="amd64" + ;; + aarch64) + architecture="arm64" + ;; + esac + + # Need to install GOLANG using apt for making deb + # Remove existing go installation (goenv utility) + - rm -rf /root/.goenv/ + + build: + commands: + - echo "Building agent deb" + - AGENT_VERSION=$(cat VERSION) + - ECS_AGENT_DEB="amazon-ecs-init_${AGENT_VERSION}-1_${architecture}.deb" + - ECS_AGENT_DEB_TAR="amazon-ecs-init_${AGENT_VERSION}-1.debian.tar.xz" + - echo $(pwd) + + # Directory/GOPATH restructuring needed for CodePipeline + - cd ../.. + - GITHUBUSERNAME=$(ls) + - mkdir -p src/github.com/ + - mv $GITHUBUSERNAME src/github.com/aws + - export GOPATH=$GOPATH:$(pwd) + - cd src/github.com/aws/amazon-ecs-agent + + # Build agent tars + - GO111MODULE=auto + - sudo apt-get update -y + - sudo apt install -y make + - sudo apt install -y dpkg-dev + - sudo apt install -y devscripts + - sudo apt install -y debhelper + - sudo apt install -y golang + - which go + - go version + - make generic-deb-integrated + +artifacts: + files: + - $ECS_AGENT_DEB + - $ECS_AGENT_DEB_TAR diff --git a/buildspecs/merge-build.yml b/buildspecs/merge-build.yml index 1c27ab1449b..843356636e4 100644 --- a/buildspecs/merge-build.yml +++ b/buildspecs/merge-build.yml @@ -5,6 +5,7 @@ env: exported-variables: - CODEBUILD_BUILD_ID - ECS_AGENT_TAR + - ECS_AGENT_RPM phases: install: @@ -41,9 +42,10 @@ phases: build: commands: - - echo Building agent image + - echo "Building agent image" - AGENT_VERSION=$(cat VERSION) - ECS_AGENT_TAR="ecs-agent-v${AGENT_VERSION}.tar" + - ECS_AGENT_RPM="amazon-ecs-init-${AGENT_VERSION}-1.x86_64.rpm" - echo $(pwd) # Directory/GOPATH restructuring needed for CodePipeline @@ -57,13 +59,18 @@ phases: # Build agent tars - GO111MODULE=auto - make dockerfree-agent-image + - make generic-rpm-integrated + - ls + # Rename artifacts for architecture - | if [[ $architecture == "arm64" ]] ; then mv $ECS_AGENT_TAR "ecs-agent-arm64-v${AGENT_VERSION}.tar" + ECS_AGENT_RPM="amazon-ecs-init-${AGENT_VERSION}-1.aarch64.rpm" ECS_AGENT_TAR="ecs-agent-arm64-v${AGENT_VERSION}.tar" fi artifacts: files: - $ECS_AGENT_TAR + - $ECS_AGENT_RPM diff --git a/buildspecs/pr-build-ubuntu.yml b/buildspecs/pr-build-ubuntu.yml new file mode 100644 index 00000000000..2d40498fe35 --- /dev/null +++ b/buildspecs/pr-build-ubuntu.yml @@ -0,0 +1,63 @@ +version: 0.2 + +env: + variables: + # Github username of the forked repo on which to make builds + GITHUBUSERNAME: prateekchaudhry + +phases: + install: + commands: + - architecture="" + # Same buildspec for different architectures - detect the architecture here and rename the artifacts accordingly + - case $(uname -m) in + x86_64) + architecture="amd64" + ;; + aarch64) + architecture="arm64" + ;; + esac + + # Need to install GOLANG using apt for making deb + # Remove existing go installation (goenv utility) + - rm -rf /root/.goenv/ + + build: + commands: + - echo "build_id = $CODEBUILD_LOG_PATH" 2>&1 | tee -a $BUILD_LOG + - echo "Building agent deb" 2>&1 | tee -a $BUILD_LOG + - AGENT_VERSION=$(cat VERSION) + - ECS_AGENT_DEB="amazon-ecs-init_${AGENT_VERSION}-1_${architecture}.deb" + - ECS_AGENT_DEB_TAR="amazon-ecs-init_${AGENT_VERSION}-1.debian.tar.xz" + - echo $(pwd) 2>&1 | tee -a $BUILD_LOG + + # Path readjustment for codebuild testing with fork and setting GOPATH appropriately + - cd ../../../.. + - export GOPATH=$GOPATH:$(pwd) + - cd src/github.com + - mv $GITHUBUSERNAME aws + - cd aws/amazon-ecs-agent + + # Building agent deb + - GO111MODULE=auto + - apt-get update -y | tee -a $BUILD_LOG + - apt install -y make | tee -a $BUILD_LOG + - apt install -y dpkg-dev | tee -a $BUILD_LOG + - apt install -y devscripts | tee -a $BUILD_LOG + - apt install -y debhelper | tee -a $BUILD_LOG + - apt install -y golang | tee -a $BUILD_LOG + - which go + - go version + - make generic-deb-integrated 2>&1 | tee -a $BUILD_LOG + - ls | tee -a $BUILD_LOG + + post_build: + commands: + +artifacts: + files: + - $ECS_AGENT_DEB + - $ECS_AGENT_DEB_TAR + - $BUILD_LOG + name: $CODEBUILD_RESOLVED_SOURCE_VERSION diff --git a/buildspecs/pr-build.yml b/buildspecs/pr-build.yml index 885b1be2300..7018f71672c 100644 --- a/buildspecs/pr-build.yml +++ b/buildspecs/pr-build.yml @@ -19,13 +19,13 @@ phases: ;; esac - # Set up proper go version using goenv utility (pre-installed in CodeBuild). Need to use this because default images come with 1.14.x + # Set up proper go version - GOVERSION="$(cat GO_VERSION)" - BUILD_LOG="build_${architecture}.log" - GOLANG_TAR="go${GOVERSION}.linux-${architecture}.tar.gz" # Need to install GOLANG explicitly as required versions do not come preinstalled - # Remove existing go installation and install downloaded binaries + # Remove existing go installation (goenv utility) and install downloaded binaries - rm -rf /root/.goenv/ - wget -O /tmp/${GOLANG_TAR} https://storage.googleapis.com/golang/${GOLANG_TAR} | tee $BUILD_LOG - tar -C /usr/local -xzf /tmp/${GOLANG_TAR} | tee -a $BUILD_LOG @@ -42,9 +42,10 @@ phases: commands: - go version - echo "build_id = $CODEBUILD_LOG_PATH" 2>&1 | tee -a $BUILD_LOG - - echo Building agent image 2>&1 | tee -a $BUILD_LOG + - echo "Building agent image" 2>&1 | tee -a $BUILD_LOG - AGENT_VERSION=$(cat VERSION) - ECS_AGENT_TAR="ecs-agent-v${AGENT_VERSION}.tar" + - ECS_AGENT_RPM="amazon-ecs-init-${AGENT_VERSION}-1.x86_64.rpm" - echo $(pwd) 2>&1 | tee -a $BUILD_LOG # Path readjustment for codebuild testing with fork and setting GOPATH appropriately @@ -57,10 +58,13 @@ phases: # Building agent tars - GO111MODULE=auto - make dockerfree-agent-image 2>&1 | tee -a $BUILD_LOG + - make generic-rpm-integrated 2>&1 | tee -a $BUILD_LOG + - ls # Rename artifacts for architecture - | if [[ $architecture == "arm64" ]] ; then mv $ECS_AGENT_TAR "ecs-agent-arm64-v${AGENT_VERSION}.tar" + ECS_AGENT_RPM="amazon-ecs-init-${AGENT_VERSION}-1.aarch64.rpm" ECS_AGENT_TAR="ecs-agent-arm64-v${AGENT_VERSION}.tar" fi @@ -70,6 +74,7 @@ phases: artifacts: files: - $ECS_AGENT_TAR + - $ECS_AGENT_RPM - $BUILD_LOG name: $CODEBUILD_RESOLVED_SOURCE_VERSION diff --git a/buildspecs/signing.yml b/buildspecs/signing.yml index cdcd6ba2542..0dc13cc3aab 100644 --- a/buildspecs/signing.yml +++ b/buildspecs/signing.yml @@ -47,12 +47,19 @@ phases: - gpg --allow-secret-key-import --import private.gpg # remove the private key file because we don't want it to be packaged with the artifacts - rm private.gpg - # Sign the amd tar (this is a secondary source so we have to do some copying) + # Sign the amd tar and rpm (this is a secondary source so we have to do some copying) - cp "$CODEBUILD_SRC_DIR_AmdBuildArtifact/$ECS_AGENT_AMD_TAR" $ECS_AGENT_AMD_TAR - source /tmp/functions.sh && sign_file $ECS_AGENT_AMD_TAR - # Sign the arm tar (this is a secondary source so we have to do some copying) + - cp "$CODEBUILD_SRC_DIR_AmdBuildArtifact/$ECS_AGENT_AMD_RPM" $ECS_AGENT_AMD_RPM + - source /tmp/functions.sh && sign_file $ECS_AGENT_AMD_RPM + # Sign the arm tar and rpm (this is a secondary source so we have to do some copying) - cp "$CODEBUILD_SRC_DIR_ArmBuildArtifact/$ECS_AGENT_ARM_TAR" $ECS_AGENT_ARM_TAR - source /tmp/functions.sh && sign_file $ECS_AGENT_ARM_TAR + - cp "$CODEBUILD_SRC_DIR_ArmBuildArtifact/$ECS_AGENT_ARM_RPM" $ECS_AGENT_ARM_RPM + - source /tmp/functions.sh && sign_file $ECS_AGENT_ARM_RPM + # Sign the amd deb (this is a secondary source so we have to do some copying) + - cp "$CODEBUILD_SRC_DIR_UbuntuAmdBuildArtifact/$ECS_AGENT_UBUNTU_AMD_DEB" $ECS_AGENT_UBUNTU_AMD_DEB + - source /tmp/functions.sh && sign_file $ECS_AGENT_UBUNTU_AMD_DEB # Clean up the key just in case - source /tmp/functions.sh && delete_all_secret_keys # validate that the keychain is empty @@ -62,5 +69,11 @@ artifacts: files: - $ECS_AGENT_AMD_TAR - '$ECS_AGENT_AMD_TAR.asc' + - $ECS_AGENT_AMD_RPM + - '$ECS_AGENT_AMD_RPM.asc' - $ECS_AGENT_ARM_TAR - '$ECS_AGENT_ARM_TAR.asc' + - $ECS_AGENT_ARM_RPM + - '$ECS_AGENT_ARM_RPM.asc' + - $ECS_AGENT_UBUNTU_AMD_DEB + - '$ECS_AGENT_UBUNTU_AMD_DEB.asc' diff --git a/packaging/generic-deb-integrated/debian/rules b/packaging/generic-deb-integrated/debian/rules index b880b742a30..f3144cb7d51 100755 --- a/packaging/generic-deb-integrated/debian/rules +++ b/packaging/generic-deb-integrated/debian/rules @@ -1,14 +1,21 @@ #!/usr/bin/make -f # -*- makefile -*- +# version is the "init version" VERSION := $(shell dpkg-parsechangelog -S Version) +# agent_version replaces the `-` with a space then takes the first "word" in the resulting "list" +# this essentially removes the version suffix `-1` using the limited makefile syntax +AGENT_VERSION := $(word 1, $(subst -, ,$(VERSION))) -# Uncomment this to turn on verbose mode. export DH_VERBOSE=1 %: dh $@ override_dh_auto_build: + ./scripts/get-host-certs + ./scripts/build-cni-plugins + ./scripts/build-integrated true "" false true + ./scripts/build-agent-image ./scripts/gobuild.sh debian clean: @@ -16,7 +23,7 @@ clean: rm -f amazon-ecs-init override_dh_auto_install: - cp ecs-agent.tar debian/amazon-ecs-init/var/cache/ecs/ecs-agent-v${VERSION}.tar + cp ./ecs-agent-v${AGENT_VERSION}.tar debian/amazon-ecs-init/var/cache/ecs/ecs-agent-v${VERSION}.tar echo "2" >debian/amazon-ecs-init/var/cache/ecs/state ln -s "/var/cache/ecs/ecs-agent-v${VERSION}.tar" debian/amazon-ecs-init/var/cache/ecs/ecs-agent.tar dh_installsystemd --no-start --no-enable --name=ecs diff --git a/packaging/generic-deb-integrated/debian/source/include-binaries b/packaging/generic-deb-integrated/debian/source/include-binaries index 551a0cdb07d..a8618c05567 100644 --- a/packaging/generic-deb-integrated/debian/source/include-binaries +++ b/packaging/generic-deb-integrated/debian/source/include-binaries @@ -1 +1,2 @@ -ecs-agent.tar +misc/pause-container/pause-image-tar-files/amazon-ecs-pause-arm64.tar +misc/pause-container/pause-image-tar-files/amazon-ecs-pause-amd64.tar diff --git a/packaging/generic-deb-integrated/debian/source/options b/packaging/generic-deb-integrated/debian/source/options new file mode 100644 index 00000000000..9c93fb2adbb --- /dev/null +++ b/packaging/generic-deb-integrated/debian/source/options @@ -0,0 +1,4 @@ +# Automatically add upstream changes to the quilt overlay. +# http://manpages.ubuntu.com/manpages/trusty/man1/dpkg-source.1.html +# This supports reusing the orig.tar.gz for debian increments. +auto-commit diff --git a/packaging/generic-rpm-integrated/amazon-ecs-init.spec b/packaging/generic-rpm-integrated/amazon-ecs-init.spec index 4becc9c0a07..ecd3a43ca13 100644 --- a/packaging/generic-rpm-integrated/amazon-ecs-init.spec +++ b/packaging/generic-rpm-integrated/amazon-ecs-init.spec @@ -31,7 +31,6 @@ Source2: amazon-ecs-volume-plugin.service Source3: amazon-ecs-volume-plugin.socket BuildRequires: systemd -BuildRequires: glibc-static Requires: systemd Requires: iptables Requires: procps @@ -45,7 +44,6 @@ required routes among its preparation steps. %setup -c %build -./scripts/build-pause ./scripts/get-host-certs ./scripts/build-cni-plugins ./scripts/build-integrated true "" false true diff --git a/scripts/gobuild.sh b/scripts/gobuild.sh index 605a66ecbde..e495d656b63 100755 --- a/scripts/gobuild.sh +++ b/scripts/gobuild.sh @@ -17,6 +17,7 @@ set -e export TOPWD="$(pwd)" export BUILDDIR="$(mktemp -d)" export SRCPATH="${BUILDDIR}/src/github.com/aws/amazon-ecs-agent" +export GOPATH="${TOPWD}:${BUILDDIR}" export GO111MODULE="auto" if [ -d "${TOPWD}/.git" ]; then