Skip to content

Commit a8e5e1f

Browse files
SuperStrongDinosaurSuperStrongDinosaur
authored
Merge branch 'main' into restart_host_orchestrator_endpoint
2 parents 31685a9 + 8284e22 commit a8e5e1f

File tree

22 files changed

+635
-132
lines changed

22 files changed

+635
-132
lines changed

‎.github/workflows/cloud-init-image-for-cuttlefish.yaml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
name: Workflow for Cloud init image for Cuttlefish
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- 'cloud-init-image-for-cuttlefish/**'
7+
push:
8+
branchs:
9+
- '**'
10+
11+
jobs:
12+
build-cidata-iso-job:
13+
runs-on: ubuntu-22.04
14+
container:
15+
image: debian@sha256:9258a75a7e4323c9e5562b361effc84ee747920116d8adfc98a465a5cdc9150e # debian:bookworm-20250407 (amd64)
16+
defaults:
17+
run:
18+
working-directory: ./cloud-init-image-for-cuttlefish
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@v4
22+
- name: Setting up build environment
23+
run: |
24+
apt-get update
25+
apt-get upgrade -y
26+
apt-get install -y sudo
27+
apt-get install -y ca-certificates
28+
apt-get install -y cloud-init
29+
apt-get install -y xorriso genisoimage
30+
- name: Check user-data
31+
run: |
32+
./check_user_data.sh
33+
- name: Create gigabyte-cidata.iso image
34+
run: |
35+
./gen_cidata_iso.sh
36+
- name: Upload artifacts
37+
uses: actions/upload-artifact@v4
38+
with:
39+
name: cloud-init-image-for-cuttlefish-artifacts
40+
path: cloud-init-image-for-cuttlefish/gigabyte-cidata.iso
41+
42+
test-qemu-cloud-arm64-cidata-job:
43+
needs: build-cidata-iso-job
44+
runs-on: ubuntu-22.04
45+
container:
46+
image: debian:trixie # debian:trixie (amd64)
47+
defaults:
48+
run:
49+
working-directory: ./cloud-init-image-for-cuttlefish
50+
env:
51+
TEST_DISK_SIZE: "10G"
52+
steps:
53+
- name: Checkout repository
54+
uses: actions/checkout@v4
55+
- name: Download artifacts
56+
uses: actions/download-artifact@v4
57+
with:
58+
name: cloud-init-image-for-cuttlefish-artifacts
59+
- name: Prepare test environment
60+
run: |
61+
apt-get update
62+
apt-get upgrade -y
63+
apt-get install -y sudo
64+
apt-get install -y debconf-utils
65+
echo "tzdata tzdata/Areas select Etc" | debconf-set-selections -v
66+
echo "tzdata tzdata/Zones/Etc select UTC" | debconf-set-selections -v
67+
DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata
68+
dpkg-reconfigure --frontend noninteractive tzdata
69+
echo "pbuilder pbuilder/mirrorsite string http://deb.debian.org/debian/" | debconf-set-selections -v
70+
DEBIAN_FRONTEND=noninteractive apt-get install -y pbuilder
71+
dpkg-reconfigure --frontend noninteractive pbuilder
72+
apt-get install -y util-linux util-linux-extra
73+
apt-get install -y xz-utils
74+
apt-get install -y qemu-system-arm
75+
apt-get install -y expect
76+
apt-get install -y e2tools e2fsprogs
77+
apt-get install -y guestfish
78+
apt-get install -y lvm2
79+
apt-get install -y ubuntu-dev-tools
80+
apt-get install -y u-boot-qemu
81+
apt-get install -y wget curl
82+
apt-get install -y openssh-client sshpass
83+
apt-get install -y screen
84+
- name: Download Debian
85+
run: |
86+
./download_debian.sh
87+
- name: Download Cuttlefish
88+
run: |
89+
../gigabyte-ampere-cuttlefish-installer/utils/download-ci-cf.sh
90+
- name: Move artifacts to current working directory.
91+
run: |
92+
mv ../gigabyte-cidata.iso .
93+
- name: Cloud image first boot with qemu
94+
run: |
95+
screen -d -m -L -Logfile console_001.log ./run_qemu.sh
96+
while ! egrep "[^[:space:]]+[[:space:]]login:" console_001.log; do sleep 30; done
97+
cp -f console_001.log console_001_p1.log
98+
CONSOLELINES=$(cat console_001_p1.log | wc -l)
99+
cat console_001_p1.log
100+
grep "login:" console_001_p1.log > /dev/null
101+
echo "CONSOLELINES=${CONSOLELINES}" >> $GITHUB_ENV
102+
- name: Wait for post install script finish
103+
run: |
104+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'while [ ! -e /var/log/custom-startup.log ]; do sleep 30; echo "Check custom-startup.log"; done'
105+
sshpass -p cuttlefish scp -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -P 33322 vsoc-01@localhost:/var/log/custom-startup.log ./custom-startup.log
106+
CUSTOM_LOG_LINES=$(wc -l < ./custom-startup.log)
107+
cat ./custom-startup.log
108+
while ! grep "VM successful" ./custom-startup.log; do sleep 30; sshpass -p cuttlefish scp -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -P 33322 vsoc-01@localhost:/var/log/custom-startup.log ./custom-startup.log; tail -n +${CUSTOM_LOG_LINES} ./custom-startup.log; CUSTOM_LOG_LINES=$(wc -l < ./custom-startup.log); done
109+
- name: Deploy cuttlefish
110+
run: |
111+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'echo cuttlefish | sudo -S -k apt-get install -y unzip'
112+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'mkdir cf'
113+
sshpass -p cuttlefish scp -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -P 33322 cvd-host_package.tar.gz vsoc-01@localhost:/home/vsoc-01/
114+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'cd cf; tar -xvf ../cvd-host_package.tar.gz'
115+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'rm -f cvd-host_package.tar.gz'
116+
sshpass -p cuttlefish scp -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -P 33322 aosp_cf_arm64*_phone-*.zip vsoc-01@localhost:/home/vsoc-01/
117+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'cd cf; unzip ../aosp_cf_arm64*_phone-*.zip'
118+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'rm -f aosp_cf_arm64*_phone-*.zip'
119+
- name: Run cuttlefish command line
120+
run: |
121+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'cd cf; HOME=$PWD ./bin/launch_cvd -help' || true
122+
- name: Run apt-cache policy
123+
run: |
124+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'apt-cache policy'
125+
- name: Test for lzop
126+
run: |
127+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'lzop -V'
128+
- name: Test for Google NTP server
129+
run: |
130+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'ntpq -p'
131+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'ntpq -p' | grep 'time1.google' && CHECK_GOOGLE_TIME_SERVER=1
132+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'ntpq -p' | grep '.GOOG.' && CHECK_GOOGLE_TIME_SERVER=1
133+
if [ x"${CHECK_GOOGLE_TIME_SERVER}" != x"1" ]; then echo "Google Time Servers not found!"; false; fi
134+
- name: Test for ulimit
135+
run: |
136+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'ulimit -a'
137+
test $(sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'ulimit -n') -ge 2048
138+
- name: Test for iptables command
139+
run: |
140+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'update-alternatives --display iptables' | grep "link currently points to /usr/sbin/iptables-legacy"
141+
- name: Shutdown qemu
142+
run: |
143+
sshpass -p cuttlefish ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null" -p 33322 vsoc-01@localhost 'echo cuttlefish | sudo -S -k shutdown -h 1'
144+
while ! grep "reboot: Power down" console_001.log; do sleep 30; done
145+
tail -n +"${CONSOLELINES}" console_001.log

‎base/cvd/cuttlefish/common/libs/utils/files.cpp‎

Lines changed: 30 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -154,24 +154,21 @@ Result<void> HardLinkDirecoryContentsRecursively(
154154
const std::string& source, const std::string& destination) {
155155
CF_EXPECTF(IsDirectory(source), "Source '{}' is not a directory", source);
156156

157-
// TODO: b/471069557 - diagnose need
158-
Result<void> unused = EnsureDirectoryExists(destination, 0755);
159-
160-
const std::function<bool(const std::string&)> linker =
161-
[&source, &destination](const std::string& filepath) mutable {
162-
std::string src_path = filepath;
163-
std::string dst_path =
164-
destination + "/" + filepath.substr(source.size() + 1);
165-
if (IsDirectory(src_path)) {
166-
// TODO: b/471069557 - diagnose need
167-
Result<void> unused = EnsureDirectoryExists(dst_path);
168-
return true;
169-
}
170-
bool overwrite_existing = true;
171-
Result<std::string> result =
172-
CreateHardLink(src_path, dst_path, overwrite_existing);
173-
return result.ok();
174-
};
157+
CF_EXPECT(EnsureDirectoryExists(destination, 0755));
158+
159+
auto linker = [&source, &destination](
160+
const std::string& filepath) mutable -> Result<void> {
161+
const std::string src_path = filepath;
162+
const std::string dst_path =
163+
destination + "/" + filepath.substr(source.size() + 1);
164+
if (IsDirectory(src_path)) {
165+
CF_EXPECT(EnsureDirectoryExists(dst_path));
166+
return {};
167+
}
168+
const bool overwrite_existing = true;
169+
CF_EXPECT(CreateHardLink(src_path, dst_path, overwrite_existing));
170+
return {};
171+
};
175172
CF_EXPECT(WalkDirectory(source, linker));
176173

177174
return {};
@@ -617,34 +614,29 @@ bool FileIsSocket(const std::string& path) {
617614
Result<std::string> FindFile(const std::string& path,
618615
const std::string& target_name) {
619616
std::string ret;
620-
auto res = WalkDirectory(
621-
path, [&ret, &target_name](const std::string& filename) mutable {
622-
if (android::base::Basename(filename) == target_name) {
623-
ret = filename;
624-
}
625-
return true;
626-
});
627-
if (!res.ok()) {
628-
return "";
629-
}
617+
auto callback = [&ret, &target_name](
618+
const std::string& filename) mutable -> Result<void> {
619+
if (android::base::Basename(filename) == target_name) {
620+
ret = filename;
621+
}
622+
return {};
623+
};
624+
CF_EXPECT(WalkDirectory(path, callback));
625+
CF_EXPECTF(!ret.empty(), "No file matching '{}' found in '{}'", target_name,
626+
path);
630627
return ret;
631628
}
632629

633630
// Recursively enumerate files in |dir|, and invoke the callback function with
634631
// path to each file/directory.
635-
Result<void> WalkDirectory(
636-
const std::string& dir,
637-
const std::function<bool(const std::string&)>& callback) {
638-
const auto files = CF_EXPECT(DirectoryContents(dir));
639-
for (const auto& filename : files) {
632+
Result<void> WalkDirectory(const std::string& dir,
633+
const WalkDirectoryCallback& callback) {
634+
for (const std::string& filename : CF_EXPECT(DirectoryContents(dir))) {
640635
auto file_path = dir + "/";
641636
file_path.append(filename);
642-
callback(file_path);
637+
CF_EXPECT(callback(file_path));
643638
if (DirectoryExists(file_path)) {
644-
auto res = WalkDirectory(file_path, callback);
645-
if (!res.ok()) {
646-
return res;
647-
}
639+
CF_EXPECT(WalkDirectory(file_path, callback));
648640
}
649641
}
650642
return {};

‎base/cvd/cuttlefish/common/libs/utils/files.h‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,10 @@ FileSizes SparseFileSizes(const std::string& path);
9999
Result<std::string> FindFile(const std::string& path,
100100
const std::string& target_name);
101101

102-
Result<void> WalkDirectory(
103-
const std::string& dir,
104-
const std::function<bool(const std::string&)>& callback);
102+
using WalkDirectoryCallback = std::function<Result<void>(const std::string&)>;
103+
104+
Result<void> WalkDirectory(const std::string& dir,
105+
const WalkDirectoryCallback& callback);
105106

106107
// parameter to EmulateAbsolutePath
107108
struct InputPathForm {

‎base/cvd/cuttlefish/host/commands/assemble_cvd/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,9 @@ cf_cc_library(
123123
"//cuttlefish/io:chroot",
124124
"//cuttlefish/io:concat",
125125
"//cuttlefish/io:copy",
126+
"//cuttlefish/io:in_memory",
126127
"//cuttlefish/io:length",
128+
"//cuttlefish/io:lz4_legacy",
127129
"//cuttlefish/io:native_filesystem",
128130
"//cuttlefish/io:shared_fd",
129131
"//cuttlefish/result",

‎base/cvd/cuttlefish/host/commands/assemble_cvd/boot_image_utils.cc‎

Lines changed: 14 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
#include "cuttlefish/io/copy.h"
4646
#include "cuttlefish/io/io.h"
4747
#include "cuttlefish/io/length.h"
48+
#include "cuttlefish/io/lz4_legacy.h"
4849
#include "cuttlefish/io/native_filesystem.h"
4950
#include "cuttlefish/io/shared_fd.h"
5051
#include "cuttlefish/result/result.h"
@@ -161,29 +162,20 @@ Result<void> PackRamdisk(const std::string& ramdisk_stage_dir,
161162

162163
Result<void> UnpackRamdisk(const std::string& original_ramdisk_path,
163164
const std::string& ramdisk_stage_dir) {
164-
int success = 0;
165-
if (IsCpioArchive(original_ramdisk_path)) {
166-
CF_EXPECTF(Copy(original_ramdisk_path, original_ramdisk_path + kCpioExt),
167-
"Failed to copy '{}' to '{}'", original_ramdisk_path,
168-
original_ramdisk_path + kCpioExt);
169-
} else {
170-
SharedFD output_fd = SharedFD::Open(original_ramdisk_path + kCpioExt,
171-
O_CREAT | O_RDWR | O_TRUNC, 0644);
172-
CF_EXPECTF(output_fd->IsOpen(), "Failed to open '{}': '{}'",
173-
original_ramdisk_path + kCpioExt, output_fd->StrError());
174-
175-
success = Command(HostBinaryPath("lz4"))
176-
.AddParameter("-c")
177-
.AddParameter("-d")
178-
.AddParameter("-l")
179-
.AddParameter(original_ramdisk_path)
180-
.RedirectStdIO(Subprocess::StdIOChannel::kStdOut, output_fd)
181-
.Start()
182-
.Wait();
183-
CF_EXPECT_EQ(
184-
success, 0,
185-
"Unable to run lz4 on file '" << original_ramdisk_path << "'.");
165+
NativeFilesystem fs;
166+
std::unique_ptr<Reader> ramdisk_input = CF_EXPECT(fs.OpenReadOnly(original_ramdisk_path));
167+
CF_EXPECT(ramdisk_input.get());
168+
169+
const std::string output_path = original_ramdisk_path + kCpioExt;
170+
(void) fs.DeleteFile(output_path);
171+
std::unique_ptr<WriterSeeker> output = CF_EXPECT(fs.CreateFile(output_path));
172+
173+
if (!IsCpioArchive(original_ramdisk_path)) {
174+
ramdisk_input = CF_EXPECT(Lz4LegacyReader(std::move(ramdisk_input)));
186175
}
176+
CF_EXPECTF(Copy(*ramdisk_input, *output), "Failed to copy '{}' to '{}'",
177+
original_ramdisk_path, output_path);
178+
187179
CF_EXPECT(EnsureDirectoryExists(ramdisk_stage_dir));
188180

189181
SharedFD input = SharedFD::Open(original_ramdisk_path + kCpioExt, O_RDONLY);

‎base/cvd/cuttlefish/host/commands/assemble_cvd/vendor_dlkm_utils.cc‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
#include "absl/strings/str_cat.h"
3030
#include "absl/strings/str_split.h"
3131
#include "android-base/file.h"
32-
#include "android-base/stringprintf.h"
3332
#include "android-base/strings.h"
3433
#include "fmt/format.h"
3534

@@ -93,21 +92,21 @@ Result<void> WriteFsConfig(const std::string& output_path,
9392
" 0 0 755 selabel=u:object_r:rootfs:s0 capabilities=0x0\n";
9493
CF_EXPECTF(android::base::WriteStringToFd(kBeginning, fd),
9594
"Failed to write to '{}'", output_path);
96-
auto res = WalkDirectory(fs_root, [&fd, &output_path, &mount_point,
97-
&fs_root](const std::string& file_path) {
98-
const auto filename = file_path.substr(
99-
fs_root.back() == '/' ? fs_root.size() : fs_root.size() + 1);
100-
std::string fs_context = " 0 0 644 capabilities=0x0\n";
101-
if (DirectoryExists(file_path)) {
102-
fs_context = " 0 0 755 capabilities=0x0\n";
103-
}
104-
if (!android::base::WriteStringToFd(
105-
mount_point + "/" + filename + fs_context, fd)) {
106-
PLOG(ERROR) << "Failed to write to " << output_path;
107-
return false;
108-
}
109-
return true;
110-
});
95+
Result<void> res = WalkDirectory(
96+
fs_root,
97+
[&fd, &output_path, &mount_point,
98+
&fs_root](const std::string& file_path) -> Result<void> {
99+
const std::string filename = file_path.substr(
100+
fs_root.back() == '/' ? fs_root.size() : fs_root.size() + 1);
101+
const std::string fs_context = DirectoryExists(file_path)
102+
? " 0 0 755 capabilities=0x0\n"
103+
: " 0 0 644 capabilities=0x0\n";
104+
CF_EXPECTF(android::base::WriteStringToFd(
105+
mount_point + "/" + filename + fs_context, fd),
106+
"Failed to write to '{}': '{}'", output_path,
107+
StrError(errno));
108+
return {};
109+
});
111110
CF_EXPECT(std::move(res));
112111
return {};
113112
}

‎base/cvd/cuttlefish/host/commands/cvd/fetch/extract_image_contents.cc‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,16 +34,16 @@ Result<std::vector<std::string>> ExtractImageContents(
3434

3535
// The image is already uncompressed. Link or move its contents.
3636
std::vector<std::string> files;
37-
const std::function<bool(const std::string&)> file_collector =
37+
auto file_collector =
3838
[&files, &image_filepath,
39-
&target_dir](const std::string& filepath) mutable {
40-
std::string target_filepath =
41-
target_dir + "/" + filepath.substr(image_filepath.size() + 1);
42-
if (!IsDirectory(filepath)) {
43-
files.push_back(target_filepath);
44-
}
45-
return true;
46-
};
39+
&target_dir](const std::string& filepath) mutable -> Result<void> {
40+
std::string target_filepath =
41+
target_dir + "/" + filepath.substr(image_filepath.size() + 1);
42+
if (!IsDirectory(filepath)) {
43+
files.push_back(target_filepath);
44+
}
45+
return {};
46+
};
4747
CF_EXPECT(WalkDirectory(image_filepath, file_collector));
4848

4949
if (keep_archive) {
@@ -52,7 +52,6 @@ Result<std::vector<std::string>> ExtractImageContents(
5252
} else {
5353
CF_EXPECT(MoveDirectoryContents(image_filepath, target_dir));
5454
// Ignore even if removing directory fails - harmless.
55-
// TODO: b/471069557 - diagnose unused
5655
Result<void> unused = RecursivelyRemoveDirectory(image_filepath);
5756
}
5857
return files;

0 commit comments

Comments
 (0)