[go: up one dir, main page]
Google Git
Sign in
chromium / infra / infra / main / . / build / build.py
blob: 44a2ad93ac2a4fb3923d2c5db450b8117822787b [file] [log] [blame]
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1#!/usr/bin/env vpython3
vadimsh1fbb6ea2015-06-13 18:01:56[diff] [blame]2# Copyright 2015 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]5
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]6"""This script builds Go binaries and invokes CIPD client to package and upload
7them to the CIPD repository.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]8
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]9See build/packages/*.yaml for definition of packages and README.md for more
10details.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]11"""
12
13import argparse
Dan Jacques72b9da62017-04-07 03:18:37[diff] [blame]14import collections
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]15import contextlib
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]16import copy
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]17import errno
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]18import functools
vadimshb545ea02015-03-11 23:08:33[diff] [blame]19import glob
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]20import hashlib
vadimshb545ea02015-03-11 23:08:33[diff] [blame]21import json
22import os
23import platform
Vadim Shtayurab8c2b882021-02-22 23:12:04[diff] [blame]24import re
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]25import shutil
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]26import socket
vadimshb545ea02015-03-11 23:08:33[diff] [blame]27import subprocess
28import sys
29import tempfile
30
Vadim Shtayura41ecb642019-04-02 19:06:22[diff] [blame]31import yaml
32
vadimshb545ea02015-03-11 23:08:33[diff] [blame]33# Root of infra.git repository.
34ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
35
vadimsh72302362015-05-07 17:42:43[diff] [blame]36# Where to upload packages to by default.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]37PACKAGE_REPO_SERVICE = 'https://chrome-infra-packages.appspot.com'
38
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]39# Mapping of CIPD arch strings to Go build env vars.
40#
41# See https://pkg.go.dev/go.chromium.org/luci/cipd/client/cipd/ensure.
42CIPD_ARCHS = {
43 '386': {
44 'GOARCH': '386',
45 'GO386': 'sse2'
46 },
47 'amd64': {
48 'GOARCH': 'amd64',
49 'GOAMD64': 'v1'
50 },
51 'arm64': {
52 'GOARCH': 'arm64'
53 },
54 'armv6l': {
55 'GOARCH': 'arm',
56 'GOARM': '6'
57 },
58 'armv7l': {
59 'GOARCH': 'arm',
60 'GOARM': '7'
61 },
62 'loong64': {
63 'GOARCH': 'loong64'
64 },
65 'mips': {
66 'GOARCH': 'mips',
67 'GOMIPS': 'hardfloat'
68 },
69 'mips64': {
70 'GOARCH': 'mips64',
71 'GOMIPS64': 'hardfloat'
72 },
73 'mips64le': {
74 'GOARCH': 'mips64le',
75 'GOMIPS64': 'hardfloat'
76 },
77 'mipsle': {
78 'GOARCH': 'mipsle',
79 'GOMIPS': 'hardfloat'
80 },
81 'ppc64': {
82 'GOARCH': 'ppc64',
83 'GOPPC64': 'power8'
84 },
85 'ppc64le': {
86 'GOARCH': 'ppc64le',
87 'GOPPC64': 'power8'
88 },
89 'riscv64': {
90 'GOARCH': 'riscv64'
91 },
92 's390x': {
93 'GOARCH': 's390x'
94 },
95}
Vadim Shtayuracd58f782018-08-06 21:16:35[diff] [blame]96
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]97# Default values for some existing GO{ARCH} vars taken from
98# https://github.com/golang/go/blob/master/src/cmd/dist/build.go
99#
100# This simplifies some code below. This also indirectly used to know what env
101# vars affect the build.
102#
103# Note that GOARM has no predefined default value (it depends on the host
104# environment) and we always require it to be set explicitly since we don't want
105# our arm32 builds to depend on specifics of a particular builder machine.
106GO_ARCH_DEFAULTS = {
107 'GO386': 'sse2',
108 'GOAMD64': 'v1',
109 'GOARM': None,
110 'GOMIPS': 'hardfloat',
111 'GOMIPS64': 'hardfloat',
112 'GOPPC64': 'power8',
113}
Dan Jacques72b9da62017-04-07 03:18:37[diff] [blame]114
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]115# Go env vars potentially affecting the build (at least the ones we care about).
116GO_BUILD_ENV_VARS = [
117 'GOOS', 'GOARCH', 'CGO_ENABLED', 'CGO_CFLAGS', 'CGO_LDFLAGS'
118] + list(GO_ARCH_DEFAULTS)
vadimshb545ea02015-03-11 23:08:33[diff] [blame]119
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]120# All CIPD platforms that support 'go build -race'.
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]121RACE_SUPPORTED_PLATFORMS = frozenset([
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]122 'freebsd-amd64',
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]123 'linux-amd64',
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]124 'linux-arm64',
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]125 'linux-ppc64le',
126 'mac-amd64',
127 'windows-amd64',
Vadim Shtayura38969c02018-08-01 00:59:17[diff] [blame]128])
129
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]130# A package prefix => cwd to use when building this package.
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]131#
132# Can be extended via `--map-go-module` command line flag.
133DEFAULT_MODULE_MAP = {
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]134 # The luci-go module is checked out separately, use its go.mod.
135 'go.chromium.org/luci/':
136 os.path.join(ROOT, 'go', 'src', 'go.chromium.org', 'luci'),
137 # All infra packages should use go.mod in infra.git.
138 'infra/':
139 os.path.join(ROOT, 'go', 'src', 'infra'),
Gregory NISBET148ebbb2025-02-19 01:16:10[diff] [blame]140 'go.chromium.org/infra/':
141 os.path.join(ROOT, 'go', 'src', 'infra'),
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]142 # Use infra's go.mod when building goldctl.
143 'go.skia.org/infra/gold-client/cmd/goldctl':
144 os.path.join(ROOT, 'go', 'src', 'infra')
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]145}
146
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]147
148def check_output(*args, **kwargs):
149 return subprocess.check_output(*args, text=True, **kwargs)
150
151
152# This can be replaced with functools.cache once we are on Py 3.9
153def cache(f):
154 value = []
155
156 @functools.wraps(f)
157 def wrapper():
158 if not value:
159 value.append(f())
160 return value[0]
161
162 return wrapper
163
vadimshb545ea02015-03-11 23:08:33[diff] [blame]164
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]165class PackageDefException(Exception):
166 """Raised if a package definition is invalid."""
167 def __init__(self, path, msg):
168 super(PackageDefException, self).__init__('%s: %s' % (path, msg))
169
170
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]171class UnsupportedException(Exception):
172 """Raised if some combination of parameters is not supported."""
173
174
vadimshb545ea02015-03-11 23:08:33[diff] [blame]175class BuildException(Exception):
vadimshae9dde02015-06-04 23:52:37[diff] [blame]176 """Raised on errors during package build step."""
177
178
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]179class SearchException(Exception):
180 """Raised on errors during package search step."""
181
182
183class TagException(Exception):
184 """Raised on errors during package tag step."""
185
186
vadimshae9dde02015-06-04 23:52:37[diff] [blame]187class UploadException(Exception):
188 """Raised on errors during package upload step."""
vadimshb545ea02015-03-11 23:08:33[diff] [blame]189
190
Dan Jacques72b9da62017-04-07 03:18:37[diff] [blame]191class PackageDef(collections.namedtuple(
192 '_PackageDef', ('path', 'pkg_def'))):
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]193 """Represents parsed package *.yaml file."""
194
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]195 @property
196 def name(self):
197 """Returns name of YAML file (without the directory path and extension)."""
198 return os.path.splitext(os.path.basename(self.path))[0]
199
200 @property
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]201 def platforms(self):
202 """Returns a list of CIPD platforms to build the package for."""
203 return self.pkg_def.get('platforms') or []
204
205 @property
Vadim Shtayura14654762019-04-11 18:13:25[diff] [blame]206 def disabled(self):
207 """Returns True if the package should be excluded from the build."""
208 return self.pkg_def.get('disabled', False)
209
210 @property
Robert Iannucci659ad472020-12-15 21:23:27[diff] [blame]211 def update_latest_ref(self):
212 """Returns True if 'update_latest_ref' in the YAML file is set.
213
214 Defaults to True."""
215 return bool(self.pkg_def.get('update_latest_ref', True))
216
217 @property
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]218 def go_packages(self):
219 """Returns a list of Go packages that must be installed for this package."""
220 return self.pkg_def.get('go_packages') or []
221
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]222 def go_build_environ_key(self, key, cipd_platform):
223 """Looks up a key in "go_build_environ" recognizing per-platform values."""
224 val = self.pkg_def.get('go_build_environ', {}).get(key)
225
226 # Legacy key name for "cgo".
227 if val is None and key == 'cgo':
228 val = self.pkg_def.get('go_build_environ', {}).get('CGO_ENABLED')
229 if val:
230 print('DEPRECATED: replace "CGO_ENABLED" with "cgo" in %s' % self.path)
231
232 if not isinstance(val, dict):
233 return val
234
235 # Per-platform setting.
236 if cipd_platform in val:
237 return val[cipd_platform]
238
239 # Per-OS setting.
240 for k, v in val.items():
241 if cipd_platform.startswith('%s-' % k):
242 return v
243 # Support 'darwin' GOOS as a legacy value for 'mac' CIPD OS.
244 if k == 'darwin':
245 print('DEPRECATED: replace "darwin" with "mac" in %s' % self.path)
246 if k == 'darwin' and cipd_platform.startswith('mac-'):
247 return v
248
249 # No setting found for the platform.
250 return None
251
252 def cgo_enabled(self, cipd_platform):
Vadim Shtayura103e5432024-01-03 02:08:42[diff] [blame]253 """True if the package needs cgo, False to disable it.
Vadim Shtayura904b7962021-02-17 01:04:06[diff] [blame]254
Vadim Shtayura103e5432024-01-03 02:08:42[diff] [blame]255 By default cgo is disabled.
Vadim Shtayura904b7962021-02-17 01:04:06[diff] [blame]256 """
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]257 return bool(self.go_build_environ_key('cgo', cipd_platform))
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]258
259 @property
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]260 def pkg_root(self):
261 """Absolute path to a package root directory."""
262 root = self.pkg_def['root'].replace('/', os.sep)
263 if os.path.isabs(root):
264 return root
265 return os.path.abspath(os.path.join(os.path.dirname(self.path), root))
Dan Jacques72b9da62017-04-07 03:18:37[diff] [blame]266
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]267 def with_race(self, cipd_platform):
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]268 """Returns True if should build with `-race` flag.
269
270 To build with race:
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]271 - race should be enabled on the cipd_platform or in general;
272 - cgo should be enabled on cipd_platform;
273 - cipd_platform should be one of the supported platforms.
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]274 """
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]275 val = self.go_build_environ_key('race', cipd_platform)
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]276 if not val:
277 return False
278
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]279 cgo_enabled = self.cgo_enabled(cipd_platform)
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]280 if not cgo_enabled:
281 print(
282 'go build -race cannot be enabled because CGO is not enabled on %s' %
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]283 cipd_platform)
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]284 return False
285
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]286 if cipd_platform in RACE_SUPPORTED_PLATFORMS:
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]287 return True
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]288 print('go build -race is not supported on %s' % cipd_platform)
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]289 return False
290
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]291 def validate(self):
292 """Raises PackageDefException if the package definition looks invalid."""
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]293 if not self.platforms:
294 raise PackageDefException(
295 self.path,
296 'At least one platform should be specified in "platforms" section.')
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]297 for var_name in self.pkg_def.get('go_build_environ', {}):
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]298 if var_name not in ['CGO_ENABLED', 'cgo', 'race']:
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]299 raise PackageDefException(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]300 self.path, 'Unsupported go_build_environ key %s' % var_name)
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]301
Brian Rynerce888682022-06-07 04:08:59[diff] [blame]302 def preprocess(self, build_root, pkg_vars, cipd_exe, sign_id=None):
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]303 """Parses the definition and filters/extends it before passing to CIPD.
304
305 This process may generate additional files that are put into the package.
306
307 Args:
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]308 build_root: root directory for building cipd package.
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]309 pkg_vars: dict with variables passed to cipd as -pkg-var.
Brian Rynerce888682022-06-07 04:08:59[diff] [blame]310 cipd_exe: path to cipd executable.
311 sign_id: identity used for Mac codesign.
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]312
313 Returns:
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]314 Path to filtered package definition YAML.
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]315
316 Raises:
317 BuildException on error.
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]318 """
319 pkg_def = copy.deepcopy(self.pkg_def)
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]320
321 pkg_def['root'] = build_root
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]322
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]323 bat_files = [
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]324 d['file'] for d in pkg_def['data'] if d.get('generate_bat_shim')
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]325 ]
326
Brian Ryner059bcab2022-06-28 15:23:46[diff] [blame]327 def process_cipd_export(ensure_contents,
328 dest,
329 pkg_vars=pkg_vars,
330 cipd_exe=cipd_exe):
331 # Render target_platform in the ensure file.
332 ensure_contents = ensure_contents.replace('${target_platform}',
333 pkg_vars['platform'])
334 cipd_export(ensure_contents, dest, cipd_exe)
335
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]336 if 'mac_bundle' in pkg_def:
337 bundle_def = pkg_def['mac_bundle']
338 bundle = create_mac_bundle(build_root, bundle_def)
339 pkg_def['data'].append({
340 'dir':
341 os.path.relpath(bundle['root'], build_root).replace(os.sep, '/')
342 })
343
344 for d in bundle_def['data']:
Brian Ryner059bcab2022-06-28 15:23:46[diff] [blame]345 if 'file' in d:
346 file_path = render_path(d['file'], pkg_vars)
347 src = os.path.join(self.pkg_root, file_path)
348 dst = os.path.join(bundle['files_root'], d['path'],
349 os.path.basename(file_path))
350 shutil.copy(src, dst)
351 elif 'cipd_export' in d:
352 process_cipd_export(d['cipd_export'], bundle['root'])
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]353
354 if 'codesign' in bundle_def:
355 cmd = ['/usr/bin/codesign', '--deep', '--force']
356 if sign_id:
357 for k, v in bundle_def['codesign'].items():
358 cmd.extend(['--' + k, v])
359 cmd.extend(['--sign', sign_id])
360 else:
361 # Ignoring all codesign args and use ad-hoc signing for testing.
362 cmd.extend(['--sign', '-'])
363 cmd.append(bundle['root'])
364
365 print('Running %s' % ' '.join(cmd))
366 subprocess.check_call(cmd)
367
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]368 for cp in pkg_def.get('copies', ()):
smut17d64c32019-07-11 21:50:20[diff] [blame]369 plat = cp.get('platforms')
370 if plat and pkg_vars['platform'] not in plat:
371 continue
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]372 dst = os.path.join(build_root, render_path(cp['dst'], pkg_vars))
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]373 shutil.copy(os.path.join(self.pkg_root, render_path(cp['src'], pkg_vars)),
374 dst)
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]375 pkg_def['data'].append(
376 {'file': os.path.relpath(dst, build_root).replace(os.sep, '/')})
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]377 if cp.get('generate_bat_shim'):
378 bat_files.append(cp['dst'])
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]379
Brian Rynerce888682022-06-07 04:08:59[diff] [blame]380 if 'cipd_export' in pkg_def:
Brian Ryner059bcab2022-06-28 15:23:46[diff] [blame]381 process_cipd_export(pkg_def['cipd_export'], build_root)
Brian Rynerce888682022-06-07 04:08:59[diff] [blame]382
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]383 # Copy all included files into build root if not existed. This must be after
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]384 # steps generating files and before any steps referring a symbolic link.
385 for d in self.pkg_def['data']:
386 path = d.get('file') or d.get('dir')
387 if path:
388 copy_if_not_exist(self.pkg_root, build_root, path, pkg_vars)
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]389
390 if not is_targeting_windows(pkg_vars):
391 for sym in pkg_def.get('posix_symlinks', ()):
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]392 dst = os.path.join(build_root, render_path(sym['dst'], pkg_vars))
Vadim Shtayura271a0b22021-02-12 01:48:24[diff] [blame]393 try:
394 os.remove(dst)
395 except OSError:
396 pass
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]397 os.symlink(
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]398 os.path.join(build_root, render_path(sym['src'], pkg_vars)), dst)
399 pkg_def['data'].append(
400 {'file': os.path.relpath(dst, build_root).replace(os.sep, '/')})
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]401
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]402 # Generate *.bat shims when targeting Windows.
403 if is_targeting_windows(pkg_vars):
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]404 for f in bat_files:
405 # Generate actual *.bat.
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]406 bat_abs = generate_bat_shim(build_root, render_path(f, pkg_vars))
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]407 # Make it part of the package definition (use slash paths there).
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]408 pkg_def['data'].append(
409 {'file': os.path.relpath(bat_abs, build_root).replace(os.sep, '/')})
Robert Iannucciedc032732017-11-15 01:13:56[diff] [blame]410 # Stage it for cleanup.
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]411
412 # Keep generated yaml in the same directory to avoid rewriting paths.
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]413 out_path = os.path.join(build_root, self.name + '.processed_yaml')
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]414 with open(out_path, 'w') as f:
415 json.dump(pkg_def, f)
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]416 return out_path
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]417
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]418 def on_change_info(self, pkg_vars):
419 """Returns tags and path to check package changed."""
420 on_change_tags = [
421 get_on_change_tag(self.pkg_root, d, pkg_vars)
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]422 for d in self.pkg_def.get('upload_on_change', [])
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]423 ]
Brian Ryner2f011db2023-01-24 02:22:08[diff] [blame]424
425 # If any on_change tags are in use, also create one for the spec itself.
426 # This will capture changes such as `copies` or `cipd_export`.
427 if on_change_tags:
428 on_change_tags.append(
429 get_on_change_tag(
430 os.path.dirname(self.path), {'file': os.path.basename(self.path)},
431 pkg_vars))
432
Chenlin Fan040674b2021-11-11 02:50:48[diff] [blame]433 pkg_path = render_path(
434 self.pkg_def.get('package'), pkg_vars, replace_sep=False)
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]435 return on_change_tags, pkg_path
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]436
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]437
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]438class GoToolset(
439 collections.namedtuple(
440 'GoToolset',
441 [
442 'env', # env vars to assign as {str => str}
443 'env_prefixes', # paths to prepend to existing vars {str => [str]}
444 'env_suffixes', # paths to append to existing vars {str => [str]}
445 'version', # go version
446 'go_env', # full 'go env' dict captured when installing the toolset
447 ])):
448 """Represents a Go build environment.
449
450 Carries os.environ modifications necessary to activate a Go toolset.
451 """
452
453 def _apply_toolset_env(self):
454 """Modifies current os.environ to activate the Go toolset."""
455 for k, v in self.env.items():
456 if v is not None:
457 os.environ[k] = v
458 else:
459 os.environ.pop(k, None)
460 def split_path(env_val, filter_out):
461 if not env_val:
462 return []
463 return [p for p in env_val.split(os.pathsep) if p not in filter_out]
464 # env_prefixes['PATH'] is e.g. `["/.../a/bin", ".../b/bin"]. Need to prepend
465 # these paths to `PATH` to get "/.../a/bin:/.../b/bin:$PATH". Cleanup dups
466 # while at it. Same for suffixes.
467 for k, v in self.env_prefixes.items():
468 cur = split_path(os.environ.get(k, ''), v)
469 os.environ[k] = os.pathsep.join(v + cur)
470 for k, v in self.env_suffixes.items():
471 cur = split_path(os.environ.get(k, ''), v)
472 os.environ[k] = os.pathsep.join(cur + v)
473
474 @contextlib.contextmanager
475 def build_env(self, go_environ):
476 """Prepares os.environ to build Go code.
477
478 Args:
479 go_environ: instance of GoEnviron object with go related env vars.
480 """
481 orig_cwd = os.getcwd()
482 orig_environ = os.environ.copy()
483
484 # Note: the order is important, we want to allow go_environ to override
485 # env vars present in the default Go environ (in particular CGO_ENABLED).
486 self._apply_toolset_env()
487 go_environ.apply()
488
489 try:
490 yield
491 finally:
492 os.chdir(orig_cwd)
493 # Apparently 'os.environ = orig_environ' doesn't actually modify process
494 # environment, only modifications of os.environ object itself do.
495 for k, v in orig_environ.items():
496 os.environ[k] = v
497 for k in os.environ.keys():
498 if k not in orig_environ:
499 os.environ.pop(k)
500
501 def clean(self, go_environ, packages):
502 """Removes object files and executables left from building given packages.
503
504 Transitively cleans all dependencies (including stdlib!) and removes
505 executables from GOBIN. In Go modules mode this also appears to be
506 downloading modules.
507
508 Args:
509 go_environ: instance of GoEnviron object with go related env vars.
510 packages: list of go packages to clean (can include '...' patterns).
511 """
512 with self.build_env(go_environ):
513 print_go_step_title('Preparing:\n %s' % '\n '.join(packages))
514 subprocess.check_call(
515 args=['go', 'clean', '-i', '-r'] + list(packages),
516 stderr=subprocess.STDOUT)
517 # Above command is either silent (without '-x') or too verbose
518 # (with '-x'). Prefer the silent version, but add a note that it's
519 # alright.
520 print('Done.')
521
522 def install(self, go_environ, packages):
523 """Builds (and installs) Go packages into GOBIN via 'go install ...'.
524
525 Compiles and installs packages into default GOBIN, which is
Vadim Shtayurad743c4c2024-11-13 19:00:51[diff] [blame]526 <go_workspace>/bin (it is setup by the bootstrap script).
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]527
528 Args:
529 go_environ: instance of GoEnviron object with go related env vars.
530 packages: list of go packages to build (can include '...' patterns).
531 rebuild: if True, will forcefully rebuild all dependences.
532 """
533 args = [
534 'go', 'install', '-trimpath', '-ldflags=-buildid=', '-buildvcs=false',
535 '-v'
536 ]
537 if go_environ.with_race:
538 args.append('-race')
539
540 args += list(packages)
541 with self.build_env(go_environ):
542 print_go_step_title('Building:\n %s' % '\n '.join(packages))
543 subprocess.check_call(args=args, stderr=subprocess.STDOUT)
544
545 def build(self, go_environ, package, output):
546 """Builds a single Go package.
547
548 Args:
549 go_environ: instance of GoEnviron object with go related env vars.
550 package: go package to build.
551 output: where to put the resulting binary.
552 """
553 args = [
554 'go', 'build', '-trimpath', '-ldflags=-buildid=', '-buildvcs=false',
555 '-v', '-o', output
556 ]
557 if go_environ.with_race:
558 args.append('-race')
559
560 args.append(package)
561 with self.build_env(go_environ):
562 print_go_step_title('Building %s' % (package,))
563 subprocess.check_call(args=args, stderr=subprocess.STDOUT)
564
565
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]566class GoEnviron(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]567 collections.namedtuple('GoEnviron',
568 ['cipd_platform', 'cgo_enabled', 'with_race', 'cwd'])
569):
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]570 """Defines a per-package Go build environment modification.
571
572 It is applied on top of the default Go toolset environment from GoToolset
573 before building the package.
574 """
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]575
576 @staticmethod
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]577 def new(cipd_platform):
578 """Prepares to compile for the given target CIPD platform."""
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]579 return GoEnviron(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]580 cipd_platform=cipd_platform,
581 cgo_enabled=False,
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]582 with_race=False,
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]583 cwd=os.getcwd(),
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]584 )
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]585
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]586 def apply(self):
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]587 """Applies GoEnviron to the current os.environ and cwd."""
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]588 for k in GO_BUILD_ENV_VARS:
589 os.environ.pop(k, None)
590 os.environ.update(cipd_platform_to_go_env(self.cipd_platform))
591 os.environ['CGO_ENABLED'] = '1' if self.cgo_enabled else '0'
592
593 # Make sure we target our minimum supported macOS version.
594 if self.cgo_enabled and self.cipd_platform.startswith('mac-'):
595 if self.cipd_platform.endswith('amd64'):
596 min_os_version = '10.13'
597 else:
598 min_os_version = '11.0'
599 # Preserve default `-O2 -g` values for these flags, and add
600 # `-mmacosx-version-min`.
601 flags = '-O2 -g -mmacosx-version-min=%s' % min_os_version
602 os.environ['CGO_CFLAGS'] = flags
603 os.environ['CGO_LDFLAGS'] = flags
604
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]605 if self.cwd is not None:
606 os.chdir(self.cwd)
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]607
608
Chenlin Fan040674b2021-11-11 02:50:48[diff] [blame]609def render_path(p, pkg_vars, replace_sep=True):
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]610 """Renders ${...} substitutions in paths, converts them to native slash."""
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]611 for k, v in pkg_vars.items():
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]612 assert '${' not in v # just in case, to avoid recursive expansion
613 p = p.replace('${%s}' % k, v)
Chenlin Fan040674b2021-11-11 02:50:48[diff] [blame]614 if replace_sep:
Chenlin Fan60c78582021-11-23 01:02:49[diff] [blame]615 return os.path.normpath(p.replace('/', os.sep))
Chenlin Fan040674b2021-11-11 02:50:48[diff] [blame]616 return p
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]617
618
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]619def copy_if_not_exist(src_root, dst_root, path, pkg_vars):
620 """Copies a file from src_root to dst_root if it doesn't exist there."""
621 file_path = render_path(path, pkg_vars)
622 src = os.path.join(src_root, file_path)
623 dst = os.path.join(dst_root, file_path)
624 if os.path.exists(dst):
625 return
626
627 try:
628 os.makedirs(os.path.dirname(dst))
629 except OSError as e:
630 if e.errno != errno.EEXIST:
631 raise
632
633 copy_tree(src_root, src, dst)
634
635
Brian Rynerce888682022-06-07 04:08:59[diff] [blame]636def cipd_export(ensure_contents, dst_root, cipd_exe):
637 """Installs cipd_pkg with the given version tag to dst_root."""
638 args = [cipd_exe, 'export', '-ensure-file', '-', '-root', dst_root]
639 cmd = subprocess.Popen(
640 args,
641 stdin=subprocess.PIPE,
642 stderr=subprocess.STDOUT,
643 executable=cipd_exe)
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]644 out, _ = cmd.communicate(ensure_contents.encode())
Brian Rynerce888682022-06-07 04:08:59[diff] [blame]645 if cmd.returncode:
646 raise subprocess.CalledProcessError(cmd.returncode, args, output=out)
647
648
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]649def copy_tree(src_root, src, dst):
650 """Copies a directory from src to dst. If it's a symlink, convert it pointing
651 to relative path."""
652 if os.path.islink(src):
653 linkto = os.readlink(src)
654 if os.path.commonprefix([src_root, linkto]) == src_root:
655 linkto = os.path.relpath(linkto, os.path.dirname(src))
656 os.symlink(linkto, dst)
657 elif os.path.isdir(src):
658 os.mkdir(dst)
659 for name in os.listdir(src):
660 copy_tree(src_root, os.path.join(src, name), os.path.join(dst, name))
661 else:
662 shutil.copy(src, dst)
663
664
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]665def generate_bat_shim(pkg_root, target_rel):
666 """Writes a shim file side-by-side with target and returns abs path to it."""
667 target_name = os.path.basename(target_rel)
668 bat_name = os.path.splitext(target_name)[0] + '.bat'
669 base_dir = os.path.dirname(os.path.join(pkg_root, target_rel))
670 bat_path = os.path.join(base_dir, bat_name)
671 with open(bat_path, 'w') as fd:
672 fd.write('\n'.join([ # python turns \n into CRLF
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]673 '@set CIPD_EXE_SHIM="%%~dp0%s"' % (target_name,),
674 '@shift',
675 '@%CIPD_EXE_SHIM% %*',
676 '',
677 ]))
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]678 return bat_path
679
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]680
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]681def create_mac_bundle(pkg_root, bundle_def):
682 """
683 Generate the Mac Bundle structure.
684
685 something.app
686 Contents
687 Info.plist
688 MacOS
689 something
690 _CodeSignature # Generated by codesign
691 CodeResources
692 """
693 bundle_root = os.path.join(pkg_root, bundle_def['name'])
694 shutil.rmtree(bundle_root, ignore_errors=True)
695
696 contents_path = os.path.join(bundle_root, 'Contents')
697 os.makedirs(contents_path)
698
699 with open(os.path.join(contents_path, 'Info.plist'), 'w') as info_plist:
700 info_plist.write(bundle_def['info'])
701
702 files_root = os.path.join(contents_path, 'MacOS')
703 os.mkdir(files_root)
704 return {
705 'root': bundle_root,
706 'files_root': files_root,
707 }
708
709
Vadim Shtayurae2d572b2021-01-06 01:25:18[diff] [blame]710def find_cipd():
711 """Finds a CIPD client in PATH."""
712 exts = ('.exe', '.bat') if sys.platform == 'win32' else ('',)
713 for p in os.environ.get('PATH', '').split(os.pathsep):
714 base = os.path.join(p, 'cipd')
715 for ext in exts:
716 candidate = base + ext
717 if os.path.isfile(candidate):
718 return candidate
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]719 return 'cipd' + ('.exe' if sys.platform == 'win32' else '')
Vadim Shtayurae2d572b2021-01-06 01:25:18[diff] [blame]720
721
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]722def run_cipd(cipd_exe, cmd, args):
vadimshb545ea02015-03-11 23:08:33[diff] [blame]723 """Invokes CIPD, parsing -json-output result.
724
725 Args:
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]726 cipd_exe: path to cipd client binary to run.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]727 cmd: cipd subcommand to run.
728 args: list of command line arguments to pass to the subcommand.
729
730 Returns:
731 (Process exit code, parsed JSON output or None).
732 """
733 temp_file = None
734 try:
735 fd, temp_file = tempfile.mkstemp(suffix='.json', prefix='cipd_%s' % cmd)
736 os.close(fd)
737
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]738 cmd_line = [cipd_exe, cmd, '-json-output', temp_file] + list(args)
vadimshb545ea02015-03-11 23:08:33[diff] [blame]739
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]740 print('Running %s' % ' '.join(cmd_line))
vadimshb545ea02015-03-11 23:08:33[diff] [blame]741 exit_code = subprocess.call(args=cmd_line, executable=cmd_line[0])
742 try:
743 with open(temp_file, 'r') as f:
744 json_output = json.load(f)
745 except (IOError, ValueError):
746 json_output = None
747
748 return exit_code, json_output
749 finally:
750 try:
751 if temp_file:
752 os.remove(temp_file)
753 except OSError:
754 pass
755
756
757def print_title(title):
758 """Pretty prints a banner to stdout."""
vadimsh42fdb3b92015-03-15 18:56:06[diff] [blame]759 sys.stdout.flush()
760 sys.stderr.flush()
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]761 print()
762 print('-' * 80)
763 print(title)
764 print('-' * 80)
vadimshb545ea02015-03-11 23:08:33[diff] [blame]765
766
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]767def print_go_step_title(title):
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]768 """Same as 'print_title', but also appends values of GOOS, GOARCH, etc."""
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]769 go_mod = None
Vadim Shtayura692e6392021-08-02 23:06:20[diff] [blame]770 if os.environ.get('GO111MODULE') != 'off' and os.path.exists('go.mod'):
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]771 go_mod = os.path.abspath('go.mod')
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]772 go_vars = [(k, os.environ[k]) for k in GO_BUILD_ENV_VARS if k in os.environ]
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]773 if go_vars or go_mod:
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]774 title += '\n' + '-' * 80
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]775 if go_mod:
776 title += '\n go.mod: %s' % go_mod
777 for k, v in go_vars:
778 title += '\n %s=%s' % (k, v)
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]779 print_title(title)
vadimshb545ea02015-03-11 23:08:33[diff] [blame]780
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]781
Vadim Shtayurad743c4c2024-11-13 19:00:51[diff] [blame]782def bootstrap_go_toolset(go_bootstrap_script):
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]783 """Makes sure the go toolset is installed and returns it as GoToolset."""
784 print_title('Making sure Go toolset is installed')
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]785
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]786 # Do not install tools from tools.go, they are used only during development.
787 # This saves a bit of time.
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]788 bootstrap_env = os.environ.copy()
789 bootstrap_env['INFRA_GO_SKIP_TOOLS_INSTALL'] = '1'
Brian Rynerada6aee2023-02-08 02:06:43[diff] [blame]790
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]791 # bootstrap.py installs Go toolset if it is missing. It returns what changes
792 # to os.environ are necessary to use the installed toolset.
793 output = json.loads(
794 check_output(
795 args=[
796 sys.executable,
797 '-u',
Vadim Shtayurad743c4c2024-11-13 19:00:51[diff] [blame]798 go_bootstrap_script,
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]799 '-', # emit JSON with environ modification into stdout
800 ],
801 env=bootstrap_env))
802 go_toolset = GoToolset(
803 output['env'],
804 output['env_prefixes'],
805 output['env_suffixes'],
806 None, # don't know the version yet
807 None, # don't know the full env yet
808 )
vadimshb545ea02015-03-11 23:08:33[diff] [blame]809
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]810 with go_toolset.build_env(GoEnviron.new(get_host_cipd_platform())):
811 # This would be something like "go version go1.15.8 darwin/amd64".
812 output = check_output(['go', 'version'])
813 print(output.strip())
814 print()
vadimshb545ea02015-03-11 23:08:33[diff] [blame]815
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]816 # We want only "go1.15.8" part.
817 version = re.match(r'go version (go[\d\.]+)', output).group(1)
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]818
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]819 # See https://github.com/golang/go/blob/master/src/cmd/go/env.go for format
820 # of the output.
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]821 output = check_output(['go', 'env'])
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]822 print(output.strip())
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]823 go_env = {}
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]824 for line in output.splitlines():
825 k, _, v = line.lstrip('set ').partition('=')
826 if v.startswith('"') and v.endswith('"'):
827 v = v.strip('"')
Vadim Shtayura05fe1952023-08-21 20:45:57[diff] [blame]828 elif v.startswith("'") and v.endswith("'"):
829 v = v.strip("'")
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]830 go_env[k] = v
831 assert go_env['GOBIN']
Vadim Shtayurab8c2b882021-02-22 23:12:04[diff] [blame]832
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]833 return GoToolset(go_toolset.env, go_toolset.env_prefixes,
834 go_toolset.env_suffixes, version, go_env)
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]835
836
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]837def find_main_module(module_map, pkg):
838 """Returns a path to the main module to use when building `pkg`.
839
840 Args:
841 module_map: a dict "go package prefix => directory with main module".
842 pkg: a Go package name to look up.
843 """
844 matches = set()
845 for pfx, main_dir in module_map.items():
846 if pkg.startswith(pfx):
847 matches.add(main_dir)
848 if len(matches) == 0:
849 raise BuildException(
850 'Package %r is not in the module map %s' %
851 (pkg, module_map))
852 if len(matches) > 1:
853 raise BuildException(
854 'Package %r matches multiple modules in the module map %s' %
855 (pkg, module_map))
856 return list(matches)[0]
857
858
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]859def build_go_code(go_toolset, cipd_platform, module_map, pkg_defs):
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]860 """Builds and installs all Go packages used by the given PackageDefs.
861
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]862 In the end $GOBIN will have all built binaries, and only them (regardless of
863 whether we are cross-compiling or not).
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]864
865 Args:
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]866 go_toolset: instance of GoToolset object to use in the build.
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]867 cipd_platform: target CIPD platform to build for.
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]868 module_map: a dict "go package prefix => directory with main module".
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]869 pkg_defs: list of PackageDef objects that define what to build.
870 """
Vadim Shtayura14654762019-04-11 18:13:25[diff] [blame]871 # Exclude all disabled packages.
872 pkg_defs = [p for p in pkg_defs if not p.disabled]
873
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]874 # Values for GOOS, GOARCH, etc. based on the target platform.
875 base_environ = GoEnviron.new(cipd_platform)
Vadim Shtayura91274502018-02-22 21:39:59[diff] [blame]876
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]877 # Grab a set of all go packages we need to build and install into GOBIN,
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]878 # figuring out a go environment (and cwd) they want.
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]879 go_packages = {} # go package name => GoEnviron
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]880
Brian Rynercc21a502022-12-22 23:55:53[diff] [blame]881 # Validate that all binary names are unique, since we rely on this fact.
882 bin_name_to_pkg = {}
883
884 # The name of the binary we will produce from this go package.
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]885 exe_suffix = get_package_vars(cipd_platform)['exe_suffix']
886
Brian Rynercc21a502022-12-22 23:55:53[diff] [blame]887 def binary_name(go_pkg):
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]888 return go_pkg[go_pkg.rfind('/') + 1:] + exe_suffix
Brian Rynercc21a502022-12-22 23:55:53[diff] [blame]889
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]890 for pkg_def in pkg_defs:
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]891 pkg_env = base_environ
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]892 pkg_env = pkg_env._replace(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]893 cgo_enabled=pkg_def.cgo_enabled(cipd_platform),
894 with_race=pkg_def.with_race(cipd_platform))
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]895 for name in pkg_def.go_packages:
Vadim Shtayuraae2f8fb2021-07-14 01:31:34[diff] [blame]896 pkg_env = pkg_env._replace(cwd=find_main_module(module_map, name))
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]897 if name in go_packages and go_packages[name] != pkg_env:
898 raise BuildException(
899 'Go package %s is being built in two different go environments '
900 '(%s and %s), this is not supported' %
901 (name, pkg_env, go_packages[name]))
902 go_packages[name] = pkg_env
Brian Rynercc21a502022-12-22 23:55:53[diff] [blame]903 bin_name = binary_name(name)
904 if bin_name in bin_name_to_pkg and bin_name_to_pkg[bin_name] != name:
905 raise BuildException(
906 'Go package %s produces binary name %s, which collides with '
907 'package %s' % (name, bin_name, bin_name_to_pkg[bin_name]))
908 bin_name_to_pkg[bin_name] = name
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]909
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]910 # Group packages by the environment they want.
911 packages_per_env = {} # GoEnviron => [str]
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]912 for name, pkg_env in go_packages.items():
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]913 packages_per_env.setdefault(pkg_env, []).append(name)
914
915 # Execute build command for each individual environment.
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]916 for pkg_env, to_install in sorted(packages_per_env.items()):
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]917 to_install = sorted(to_install)
918 if not to_install:
919 continue
920
921 # Make sure there are no stale files in the workspace.
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]922 go_toolset.clean(pkg_env, to_install)
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]923
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]924 if cipd_platform == get_host_cipd_platform():
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]925 # If not cross-compiling, build all Go code in a single "go install" step,
926 # it's faster that way. We can't do that when cross-compiling, since
927 # 'go install' isn't supposed to be used for cross-compilation and the
928 # toolset actively complains with "go install: cannot install
929 # cross-compiled binaries when GOBIN is set".
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]930 go_toolset.install(pkg_env, to_install)
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]931 else:
932 # Prebuild stdlib once. 'go build' calls below are discarding build
933 # results, so it's better to install as much shared stuff as possible
934 # beforehand.
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]935 go_toolset.install(pkg_env, ['std'])
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]936
937 # Build packages one by one and put the resulting binaries into GOBIN, as
938 # if they were installed there. It's where the rest of the build.py code
939 # expects them to be (see also 'root' property in package definition
940 # YAMLs).
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]941 go_bin = go_toolset.go_env['GOBIN']
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]942 for pkg in to_install:
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]943 go_toolset.build(pkg_env, pkg, os.path.join(go_bin, binary_name(pkg)))
Dan Jacques72b9da62017-04-07 03:18:37[diff] [blame]944
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]945
Vadim Shtayura41ecb642019-04-02 19:06:22[diff] [blame]946def enumerate_packages(package_def_dir, package_def_files):
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]947 """Returns a list PackageDef instances for files in build/packages/*.yaml.
948
949 Args:
vadimshb545ea02015-03-11 23:08:33[diff] [blame]950 package_def_dir: path to build/packages dir to search for *.yaml.
951 package_def_files: optional list of filenames to limit results to.
952
953 Returns:
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]954 List of PackageDef instances parsed from *.yaml files under packages_dir.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]955 """
vadimshb545ea02015-03-11 23:08:33[diff] [blame]956 paths = []
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]957 if not package_def_files:
Vadim Shtayura14654762019-04-11 18:13:25[diff] [blame]958 # All existing packages by default.
959 paths = glob.glob(os.path.join(package_def_dir, '*.yaml'))
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]960 else:
961 # Otherwise pick only the ones in 'package_def_files' list.
962 for name in package_def_files:
963 abs_path = os.path.abspath(os.path.join(package_def_dir, name))
964 if not os.path.isfile(abs_path):
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]965 raise PackageDefException(name, 'No such package definition file')
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]966 paths.append(abs_path)
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]967 # Load and validate YAMLs.
968 pkgs = []
969 for p in sorted(paths):
Vadim Shtayura41ecb642019-04-02 19:06:22[diff] [blame]970 pkg = PackageDef(p, read_yaml(p))
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]971 pkg.validate()
972 pkgs.append(pkg)
973 return pkgs
vadimshb545ea02015-03-11 23:08:33[diff] [blame]974
975
Vadim Shtayura41ecb642019-04-02 19:06:22[diff] [blame]976def read_yaml(path):
vadimshf1072a12015-06-18 21:36:13[diff] [blame]977 """Returns content of YAML file as python dict."""
Vadim Shtayura41ecb642019-04-02 19:06:22[diff] [blame]978 with open(path, 'rb') as f:
979 return yaml.safe_load(f)
vadimshf1072a12015-06-18 21:36:13[diff] [blame]980
981
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]982def cipd_platform_to_go_env(platform):
983 """Given e.g. linux-armv7l returns {GOOS: linux, GOARCH: arm, GOARM: 7}."""
984 parts = platform.split('-')
985 if len(parts) != 2:
986 raise UnsupportedException('Bad CIPD platform: %s' % platform)
987 os, arch = parts
988 env = {'GOOS': 'darwin' if os == 'mac' else os}
989 if arch not in CIPD_ARCHS:
990 raise UnsupportedException('Unrecognized CIPD architecture: %s' % arch)
991 env.update(CIPD_ARCHS[arch])
992 return env
vadimshb545ea02015-03-11 23:08:33[diff] [blame]993
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]994
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]995def go_env_to_cipd_platform(env):
996 """Given GOOS, GOARCH, etc. env vars returns the corresponding CIPD platform.
997
998 At least GOOS, GOARCH must be present in the environment. Keys not related to
999 Go are simply ignored.
1000
1001 Raises UnsupportedException if this combination doesn't match any supported
1002 CIPD architecture.
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1003 """
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1004 assert 'GOOS' in env and 'GOARCH' in env, env
1005 os = 'mac' if env['GOOS'] == 'darwin' else env['GOOS']
1006
1007 # GOARM is a special case, since it has no default value in "go build".
1008 if env['GOARCH'] == 'arm' and 'GOARM' not in env:
1009 raise UnsupportedException('GOARM is required when GOARCH=arm')
1010
1011 # Fill in `env` with default values for various go arch env vars. Do the same
1012 # for possible CIPD_ARCH values. Then find a direct match between given go env
1013 # and one of CIPD_ARCH envs. This takes care of handling cases when the caller
1014 # explicitly sets an already default value of some env var. Or if they set it
1015 # to something we don't yet support.
1016 def normalized(env):
1017 keys = ['GOARCH'] + list(GO_ARCH_DEFAULTS)
1018 return sorted(
1019 '%s=%s' % (k, env.get(k, GO_ARCH_DEFAULTS.get(k, ''))) for k in keys)
1020
1021 got = normalized(env)
1022 for arch, varz in CIPD_ARCHS.items():
1023 if got == normalized(varz):
1024 return '%s-%s' % (os, arch)
1025 raise UnsupportedException('Unsupported Go build configuration: %s' % got)
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1026
1027
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1028@cache
1029def get_host_cipd_platform():
1030 """Derives CIPD platform of the host running this script.
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1031
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1032 This completely ignores GOOS, GOARCH, GOARM, etc. It just looks at the host
1033 OS and platform using Python.
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1034 """
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1035 platform_variant = {
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]1036 'darwin': 'mac',
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1037 'linux': 'linux',
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]1038 'linux2': 'linux',
1039 'win32': 'windows',
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1040 }.get(sys.platform)
1041 if not platform_variant:
1042 raise ValueError('Unknown OS: %s' % sys.platform)
1043
Dan Jacques5377e712017-07-20 02:10:20[diff] [blame]1044 sys_arch = None
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1045 if sys.platform in ('linux', 'linux2'):
Dan Jacquese7258242017-07-26 06:03:49[diff] [blame]1046 sys_arch = get_linux_host_arch()
vadimshc9127f62015-03-12 05:06:56[diff] [blame]1047
Dan Jacques5377e712017-07-20 02:10:20[diff] [blame]1048 # If we didn't override our system architecture, identify it using "platform".
1049 sys_arch = sys_arch or platform.machine()
Robert Iannucci7cf30192023-03-21 01:56:38[diff] [blame]1050 sys_arch_lower = sys_arch.lower()
Dan Jacques5377e712017-07-20 02:10:20[diff] [blame]1051
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1052 # A set of architectures that we expect can be building packages. This doesn't
1053 # need to cover all CIPD architectures.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1054 platform_arch = {
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1055 'amd64': 'amd64',
1056 'i386': '386',
1057 'i686': '386',
1058 'x86': '386',
1059 'x86_64': 'amd64',
1060 'arm64': 'arm64',
1061 'armv6l': 'armv6l',
1062 'armv7l': 'armv6l', # we prefer to use older instruction set for builds
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1063 }.get(sys_arch_lower, sys_arch_lower)
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1064
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1065 # Most 32-bit Linux Chrome Infra bots are in fact running 64-bit kernel with
1066 # 32-bit userland. Detect this case (based on bitness of the python
1067 # interpreter) and report the bot as '386'.
1068 if (platform_variant == 'linux' and
1069 platform_arch == 'amd64' and
1070 sys.maxsize == (2 ** 31) - 1):
1071 platform_arch = '386'
1072
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1073 # E.g. 'linux-amd64'.
1074 return '%s-%s' % (platform_variant, platform_arch)
1075
1076
1077def get_linux_host_arch():
1078 """The Linux host architecture, or None if it could not be resolved."""
1079 try:
1080 # Query "dpkg" to identify the userspace architecture.
1081 return check_output(['dpkg', '--print-architecture']).strip()
1082 except OSError:
1083 # This Linux distribution doesn't use "dpkg".
1084 return None
1085
1086
1087def get_cipd_platform_from_env():
1088 """Returns CIPD platform to use by default in this build invocation.
1089
1090 This is used if `--cipd-platform` is unset (usually when running the build
1091 script locally to test stuff). It looks at `GOOS`, `GOARCH` etc and also at
1092 the host CIPD platform.
1093
1094 Note this runs before `go` is in PATH, so it can't just ask `go env`.
1095 """
1096 # Merge explicitly set GO* env vars with ones that are based on the host.
1097 # This allows e.g. setting only GOARCH=... to do cross compilation into this
1098 # arch for the host OS.
1099 merged = os.environ.copy()
1100 for k, v in cipd_platform_to_go_env(get_host_cipd_platform()).items():
1101 if not merged.get(k):
1102 merged[k] = v
1103 return go_env_to_cipd_platform(merged)
1104
1105
1106def get_package_vars(cipd_platform):
1107 """Returns a dict with variables that describe the package target environment.
1108
1109 Variables can be referenced in the package definition YAML as
1110 ${variable_name}. It allows to reuse exact same definition file for similar
1111 packages (e.g. packages with same cross platform binary, but for different
1112 platforms).
1113 """
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1114 return {
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1115 'exe_suffix': '.exe' if cipd_platform.startswith('windows-') else '',
1116 'platform': cipd_platform,
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1117 }
1118
1119
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]1120def is_targeting_windows(pkg_vars):
1121 """Returns true if 'platform' in pkg_vars indicates Windows."""
1122 return pkg_vars['platform'].startswith('windows-')
1123
1124
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1125def get_on_change_tag(root, pkg_data, pkg_vars):
1126 """Get the tag for detecting package on change"""
1127 h = hashlib.sha256()
1128 data_file = render_path(pkg_data['file'], pkg_vars)
1129 with open(os.path.join(root, data_file), 'rb') as f:
1130 for chunk in iter(lambda: f.read(h.block_size * 256), b""):
1131 h.update(chunk)
1132 return ':'.join(['on_change', data_file, h.name, h.hexdigest()])
1133
1134
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1135def build_pkg(cipd_exe, pkg_def, out_file, package_vars, sign_id=None):
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1136 """Invokes CIPD client to build a package.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1137
1138 Args:
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1139 cipd_exe: path to cipd client binary to use.
1140 pkg_def: instance of PackageDef representing this package.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1141 out_file: where to store the built package.
1142 package_vars: dict with variables to pass as -pkg-var to cipd.
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1143 sign_id: identity used for Mac codesign.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1144
1145 Returns:
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1146 {'package': <name>, 'instance_id': <hash>}
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1147
1148 Raises:
1149 BuildException on error.
1150 """
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1151 print_title('Building: %s' % os.path.basename(out_file))
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1152
1153 # Make sure not stale output remains.
1154 if os.path.isfile(out_file):
1155 os.remove(out_file)
1156
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]1157 try:
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]1158 build_root = tempfile.mkdtemp(prefix="build_py")
1159
1160 # Parse the definition and filter/extend it before passing to CIPD. This
1161 # process may generate additional files that are put into the package.
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1162 processed_yaml = pkg_def.preprocess(
Brian Rynerce888682022-06-07 04:08:59[diff] [blame]1163 build_root, package_vars, cipd_exe, sign_id=sign_id)
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]1164
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]1165 # Build the package.
1166 args = ['-pkg-def', processed_yaml]
1167 for k, v in sorted(package_vars.items()):
1168 args.extend(['-pkg-var', '%s:%s' % (k, v)])
1169 args.extend(['-out', out_file])
1170 exit_code, json_output = run_cipd(cipd_exe, 'pkg-build', args)
1171 if exit_code:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1172 print()
1173 print('FAILED! ' * 10, file=sys.stderr)
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]1174 raise BuildException('Failed to build the CIPD package, see logs')
1175
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1176 # Expected result is {'package': 'name', 'instance_id': 'hash'}
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]1177 info = json_output['result']
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1178 print('%s %s' % (info['package'], info['instance_id']))
Vadim Shtayurac42c72e2017-04-07 21:34:40[diff] [blame]1179 return info
1180 finally:
Chenlin Fan587f77c2021-11-21 23:14:08[diff] [blame]1181 shutil.rmtree(build_root, ignore_errors=True)
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1182
1183
Robert Iannucci659ad472020-12-15 21:23:27[diff] [blame]1184def upload_pkg(cipd_exe, pkg_file, service_url, tags, update_latest_ref,
1185 service_account):
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1186 """Uploads existing *.cipd file to the storage and tags it.
1187
1188 Args:
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1189 cipd_exe: path to cipd client binary to use.
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1190 pkg_file: path to *.cipd file to upload.
1191 service_url: URL of a package repository service.
1192 tags: a list of tags to attach to uploaded package instance.
Robert Iannucci659ad472020-12-15 21:23:27[diff] [blame]1193 update_latest_ref: a bool of whether or not to update the 'latest' CIPD ref
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1194 service_account: path to *.json file with service account to use.
1195
1196 Returns:
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1197 {'package': <name>, 'instance_id': <hash>}
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1198
1199 Raises:
1200 UploadException on error.
1201 """
1202 print_title('Uploading: %s' % os.path.basename(pkg_file))
1203
1204 args = ['-service-url', service_url]
1205 for tag in sorted(tags):
1206 args.extend(['-tag', tag])
Robert Iannucci659ad472020-12-15 21:23:27[diff] [blame]1207 if update_latest_ref:
1208 args.extend(['-ref', 'latest'])
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1209 if service_account:
1210 args.extend(['-service-account-json', service_account])
Vadim Shtayura2c805f12018-08-06 23:19:42[diff] [blame]1211 args.append(pkg_file)
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1212 exit_code, json_output = run_cipd(cipd_exe, 'pkg-register', args)
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1213 if exit_code:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1214 print()
1215 print('FAILED! ' * 10, file=sys.stderr)
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1216 raise UploadException('Failed to upload the CIPD package, see logs')
1217 info = json_output['result']
Vadim Shtayura6fa7bd62018-08-13 22:23:30[diff] [blame]1218 info['url'] = '%s/p/%s/+/%s' % (
1219 service_url, info['package'], info['instance_id'])
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1220 print('%s %s' % (info['package'], info['instance_id']))
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1221 return info
1222
1223
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1224def search_pkg(cipd_exe, pkg_name, service_url, tags, service_account):
1225 """Search existing cipd packages with given tags.
1226
1227 Args:
1228 cipd_exe: path to cipd client binary to use.
1229 pkg_name: name of the cipd package.
1230 service_url: URL of a package repository service.
1231 tags: tags to search in the package repository.
1232 service_account: path to *.json file with service account to use.
1233
1234 Returns:
1235 {'package': <name>, 'instance_id': <hash>}
1236
1237 Raises:
1238 SearchException on error.
1239 """
1240 print_title('Searching: %s by on_change tags %s' % (pkg_name, tags))
1241
1242 args = ['-service-url', service_url]
1243 for tag in tags:
1244 args.extend(['-tag', tag])
1245 if service_account:
1246 args.extend(['-service-account-json', service_account])
1247 args.append(pkg_name)
1248 exit_code, json_output = run_cipd(cipd_exe, 'search', args)
1249 if exit_code:
Chenlin Fan4e3daf52022-09-20 01:08:01[diff] [blame]1250 if json_output['error_code'] == 'auth_error':
1251 # Maybe the package doesn't exist
1252 return None
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1253 print()
1254 print('FAILED! ' * 10, file=sys.stderr)
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1255 raise SearchException('Failed to search the CIPD package, see logs')
1256 result = json_output['result']
1257 if result and len(result) > 1:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1258 print()
1259 print('FAILED! ' * 10, file=sys.stderr)
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1260 raise SearchException('Multiple CIPD package matched, %s', result)
1261 return result[0] if result else None
1262
1263
1264def tag_pkg(cipd_exe, pkg_name, pkg_version, service_url, tags,
Chenlin Fan4e3daf52022-09-20 01:08:01[diff] [blame]1265 update_latest_ref, service_account):
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1266 """Tag existing cipd package with given tags.
1267
1268 Args:
1269 cipd_exe: path to cipd client binary to use.
1270 pkg_name: name of the cipd package.
1271 pkg_version: version of the cipd package.
1272 service_url: URL of a package repository service.
Vadim Shtayura71c8c3c2022-09-23 00:50:16[diff] [blame]1273 tags: tags to set to the cipd package.
1274 update_latest_ref: a bool of whether or not to update the 'latest' CIPD ref.
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1275 service_account: path to *.json file with service account to use.
1276
1277 Raises:
1278 TagException on error.
1279 """
1280 print_title('Tagging: %s, %s' % (pkg_name, tags))
1281
1282 args = ['-service-url', service_url]
1283 for tag in tags:
1284 args.extend(['-tag', tag])
Chenlin Fan4e3daf52022-09-20 01:08:01[diff] [blame]1285 if update_latest_ref:
1286 args.extend(['-ref', 'latest'])
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1287 if service_account:
1288 args.extend(['-service-account-json', service_account])
1289 args.extend(['-version', pkg_version])
1290 args.append(pkg_name)
Vadim Shtayura71c8c3c2022-09-23 00:50:16[diff] [blame]1291 exit_code, _ = run_cipd(cipd_exe, 'attach', args)
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1292 if exit_code:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1293 print()
1294 print('FAILED! ' * 10, file=sys.stderr)
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1295 raise TagException('Failed to tag the CIPD package, see logs')
1296
1297
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1298def get_build_out_file(package_out_dir, pkg_def, out_files_sfx):
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1299 """Returns a path where to put built *.cipd package file.
1300
1301 Args:
1302 package_out_dir: root directory where to put *.cipd files.
1303 pkg_def: instance of PackageDef being built.
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1304 out_files_sfx: a suffix to append to the filename.
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1305 """
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1306 return os.path.join(package_out_dir, pkg_def.name + out_files_sfx + '.cipd')
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1307
1308
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1309def run(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1310 cipd_platform,
Vadim Shtayurad743c4c2024-11-13 19:00:51[diff] [blame]1311 go_bootstrap_script,
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1312 module_map,
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1313 package_def_dir,
1314 package_out_dir,
1315 package_def_files,
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1316 build,
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1317 upload,
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1318 sign_id,
vadimsh72302362015-05-07 17:42:43[diff] [blame]1319 service_url,
1320 tags,
vadimshd3001422015-05-05 00:09:02[diff] [blame]1321 service_account_json,
Robert Iannuccif4c04c42020-09-08 23:38:53[diff] [blame]1322 json_output,
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1323):
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1324 """Rebuilds python and Go universes and CIPD packages.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1325
1326 Args:
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1327 cipd_platform: a CIPD platform to build for or "" to auto-detect.
Vadim Shtayurad743c4c2024-11-13 19:00:51[diff] [blame]1328 go_bootstrap_script: a script to use to bootstrap Go environment.
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1329 module_map: a dict "go package prefix => directory with main module".
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1330 package_def_dir: path to build/packages dir to search for *.yaml.
1331 package_out_dir: where to put built packages.
1332 package_def_files: names of *.yaml files in package_def_dir or [] for all.
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1333 build: False to skip building packages (valid only when upload==True).
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1334 upload: True to also upload built packages, False just to build them.
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1335 sign_id: identity used for Mac codesign.
vadimsh72302362015-05-07 17:42:43[diff] [blame]1336 service_url: URL of a package repository service.
1337 tags: a list of tags to attach to uploaded package instances.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1338 service_account_json: path to *.json service account credential.
vadimshd3001422015-05-05 00:09:02[diff] [blame]1339 json_output: path to *.json file to write info about built packages to.
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1340
1341 Returns:
1342 0 on success, 1 or error.
1343 """
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1344 assert build or upload, 'Both build and upload are False, nothing to do'
1345
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1346 # If --cipd-platform is unset, derived it based on the host and environ. Check
1347 # all involved platforms are actually supported.
1348 try:
1349 cipd_platform = cipd_platform or get_cipd_platform_from_env()
1350 _ = cipd_platform_to_go_env(cipd_platform)
1351 _ = cipd_platform_to_go_env(get_host_cipd_platform())
1352 except UnsupportedException as exc:
1353 print(exc, file=sys.stderr)
1354 return 1
vadimshf1072a12015-06-18 21:36:13[diff] [blame]1355
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1356 # If cross-compiling, append a suffix to the *.cipd files stored in the output
1357 # directory. That way we can store many different variants of the same package
1358 # there.
1359 out_files_sfx = ''
1360 if cipd_platform != get_host_cipd_platform():
1361 out_files_sfx = '+' + cipd_platform
1362
1363 # Load all package definitions.
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]1364 try:
Vadim Shtayura41ecb642019-04-02 19:06:22[diff] [blame]1365 defs = enumerate_packages(package_def_dir, package_def_files)
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]1366 except PackageDefException as exc:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1367 print(exc, file=sys.stderr)
Vadim Shtayura2288f0c2017-12-20 04:25:04[diff] [blame]1368 return 1
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1369
1370 # Pick ones we want to build based on the CIPD platform. Log when skipping
1371 # some explicitly requested packages.
1372 packages_to_visit = []
1373 for p in defs:
1374 if cipd_platform in p.platforms:
1375 packages_to_visit.append(p)
1376 elif package_def_files:
1377 print('Skipping %s since it doesn\'t list %s in its "platforms" '
1378 'section in the YAML.' % (p.name, cipd_platform))
1379
1380 # Fail if was given an explicit list of packages to build (usually just one),
1381 # but they all were filtered out.
1382 if package_def_files and not packages_to_visit:
1383 print(
1384 'None of the requested packages match CIPD platform %s, adjust their '
1385 '"platforms" section in the YAML if necessary.' % cipd_platform,
1386 file=sys.stderr)
1387 return 1
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1388
Vadim Shtayurab8c2b882021-02-22 23:12:04[diff] [blame]1389 # Make sure we have a Go toolset and it matches the host platform we detected
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1390 # in get_host_cipd_platform() as an extra check that the environment looks
1391 # good. In theory we can use any toolset, since we are going to be setting all
1392 # GOOS, GOARCH etc env vars explicitly, enabling cross-compilation, but an
1393 # extra check won't hurt.
Vadim Shtayurad743c4c2024-11-13 19:00:51[diff] [blame]1394 go_toolset = bootstrap_go_toolset(go_bootstrap_script)
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1395 expected_host_env = cipd_platform_to_go_env(get_host_cipd_platform())
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]1396 if go_toolset.go_env['GOHOSTARCH'] != expected_host_env['GOARCH']:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1397 print(
Vadim Shtayurab8c2b882021-02-22 23:12:04[diff] [blame]1398 'Go toolset GOHOSTARCH (%s) doesn\'t match expected architecture (%s)' %
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]1399 (go_toolset.go_env['GOHOSTARCH'], expected_host_env['GOARCH']),
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1400 file=sys.stderr)
Vadim Shtayurab8c2b882021-02-22 23:12:04[diff] [blame]1401 return 1
1402
1403 # Append tags related to the build host. They are especially important when
1404 # cross-compiling: cross-compiled packages can be identified by comparing the
1405 # platform in the package name with value of 'build_host_platform' tag.
1406 tags = list(tags)
1407 tags.append('build_host_hostname:' + socket.gethostname().split('.')[0])
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1408 tags.append('build_host_platform:' + get_host_cipd_platform())
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]1409 tags.append('go_version:' + go_toolset.version)
Vadim Shtayurab8c2b882021-02-22 23:12:04[diff] [blame]1410
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1411 print_title('Overview')
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1412 print('CIPD platform:')
1413 print(' target = %s' % cipd_platform)
1414 print(' host = %s' % get_host_cipd_platform())
1415 print()
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1416 if upload:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1417 print('Service URL: %s' % service_url)
1418 print()
Vadim Shtayura0736ccb2024-10-23 21:26:55[diff] [blame]1419 print('Package definition files to process:')
Vadim Shtayura14654762019-04-11 18:13:25[diff] [blame]1420 for pkg_def in packages_to_visit:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1421 print(' %s' % pkg_def.name)
Vadim Shtayura14654762019-04-11 18:13:25[diff] [blame]1422 if not packages_to_visit:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1423 print(' <none>')
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1424 print()
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1425 print('Variables to pass to CIPD:')
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1426 package_vars = get_package_vars(cipd_platform)
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1427 for k, v in sorted(package_vars.items()):
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1428 print(' %s = %s' % (k, v))
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1429 if upload:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1430 print()
1431 print('Tags to attach to uploaded packages:')
vadimsh72302362015-05-07 17:42:43[diff] [blame]1432 for tag in sorted(tags):
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1433 print(' %s' % tag)
Vadim Shtayura14654762019-04-11 18:13:25[diff] [blame]1434 if not packages_to_visit:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1435 print()
1436 print('Nothing to do.')
vadimshf1072a12015-06-18 21:36:13[diff] [blame]1437 return 0
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1438
Vadim Shtayurae2d572b2021-01-06 01:25:18[diff] [blame]1439 # Find a CIPD client in PATH to use for building and uploading packages.
1440 print_title('CIPD client')
1441 cipd_exe = find_cipd()
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1442 print('Binary: %s' % cipd_exe)
Vadim Shtayurae2d572b2021-01-06 01:25:18[diff] [blame]1443 subprocess.check_call(['cipd', 'version'], executable=cipd_exe)
1444
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1445 # Remove old build artifacts to avoid stale files in case the script crashes
1446 # for some reason.
1447 if build:
1448 print_title('Cleaning %s' % package_out_dir)
1449 if not os.path.exists(package_out_dir):
1450 os.makedirs(package_out_dir)
1451 cleaned = False
Vadim Shtayura14654762019-04-11 18:13:25[diff] [blame]1452 for pkg_def in packages_to_visit:
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1453 out_file = get_build_out_file(package_out_dir, pkg_def, out_files_sfx)
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1454 if os.path.exists(out_file):
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1455 print('Removing stale %s' % os.path.basename(out_file))
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1456 os.remove(out_file)
1457 cleaned = True
1458 if not cleaned:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1459 print('Nothing to clean')
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1460
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1461 # Build the world.
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1462 if build:
Vadim Shtayura27949fb2024-11-13 02:29:33[diff] [blame]1463 build_go_code(go_toolset, cipd_platform, module_map, packages_to_visit)
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1464
vadimshf1072a12015-06-18 21:36:13[diff] [blame]1465 # Package it.
vadimshd3001422015-05-05 00:09:02[diff] [blame]1466 failed = []
1467 succeeded = []
Vadim Shtayura14654762019-04-11 18:13:25[diff] [blame]1468 for pkg_def in packages_to_visit:
1469 if pkg_def.disabled:
1470 print_title('Skipping building disabled %s' % pkg_def.name)
1471 continue
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1472 out_file = get_build_out_file(package_out_dir, pkg_def, out_files_sfx)
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1473 try:
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1474 info = None
1475 if build:
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1476 info = build_pkg(
1477 cipd_exe, pkg_def, out_file, package_vars, sign_id=sign_id)
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1478 if upload:
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1479 on_change_tags, pkg_path = pkg_def.on_change_info(package_vars)
1480 if on_change_tags:
1481 existed_pkg = search_pkg(cipd_exe, pkg_path, service_url,
1482 on_change_tags, service_account_json)
1483 if existed_pkg:
1484 print('Not uploading %s, since all change tags are present.'
1485 ' result: %s' % (pkg_def.name, existed_pkg))
1486 tag_pkg(
1487 cipd_exe,
1488 existed_pkg['package'],
1489 existed_pkg['instance_id'],
1490 service_url,
1491 tags,
Chenlin Fan4e3daf52022-09-20 01:08:01[diff] [blame]1492 pkg_def.update_latest_ref,
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1493 service_account_json,
1494 )
1495 succeeded.append({
1496 'pkg_def_name': pkg_def.name,
1497 'info': existed_pkg
1498 })
1499 continue
1500 tags.extend(on_change_tags)
1501
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1502 info = upload_pkg(
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1503 cipd_exe,
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1504 out_file,
1505 service_url,
1506 tags,
Robert Iannucci659ad472020-12-15 21:23:27[diff] [blame]1507 pkg_def.update_latest_ref,
Chenlin Fan1a3c8152021-11-08 23:55:51[diff] [blame]1508 service_account_json,
1509 )
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1510 assert info is not None
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1511 succeeded.append({'pkg_def_name': pkg_def.name, 'info': info})
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1512 except (BuildException, UploadException) as e:
vadimsh43ed7be2016-06-27 23:16:17[diff] [blame]1513 failed.append({'pkg_def_name': pkg_def.name, 'error': str(e)})
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1514
1515 print_title('Summary')
vadimshd3001422015-05-05 00:09:02[diff] [blame]1516 for d in failed:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1517 print('FAILED %s, see log above' % d['pkg_def_name'])
vadimshd3001422015-05-05 00:09:02[diff] [blame]1518 for d in succeeded:
Chenlin Fanca1d47f2023-03-03 00:15:42[diff] [blame]1519 print('%s %s' % (d['info']['package'], d['info']['instance_id']))
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1520
vadimshd3001422015-05-05 00:09:02[diff] [blame]1521 if json_output:
1522 with open(json_output, 'w') as f:
1523 summary = {
Chan Li6a4fc492022-08-29 20:32:31[diff] [blame]1524 'failed': failed,
1525 'succeeded': succeeded,
1526 'tags': sorted(tags),
1527 'vars': package_vars,
vadimshd3001422015-05-05 00:09:02[diff] [blame]1528 }
1529 json.dump(summary, f, sort_keys=True, indent=2, separators=(',', ': '))
1530
1531 return 1 if failed else 0
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1532
1533
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1534def main(args):
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1535 parser = argparse.ArgumentParser(description='Builds infra CIPD packages')
1536 parser.add_argument(
1537 'yamls', metavar='YAML', type=str, nargs='*',
1538 help='name of a file in build/packages/* with the package definition')
1539 parser.add_argument(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1540 '--cipd-platform',
1541 metavar='CIPD_PLATFORM',
1542 help=('CIPD platform to build packages for (if unset, derived from '
1543 'the environment)'),
1544 )
1545 parser.add_argument(
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1546 '--signing-identity',
1547 metavar='IDENTITY',
1548 dest='sign_id',
1549 default=None,
1550 help='Signing identity used for mac codesign. '
1551 'Use adhoc sign if not provided.')
1552 parser.add_argument(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1553 '--upload',
1554 action='store_true',
1555 dest='upload',
1556 default=False,
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1557 help='upload packages into the repository')
1558 parser.add_argument(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1559 '--no-rebuild',
1560 action='store_false',
1561 dest='build',
1562 default=True,
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1563 help='when used with --upload means upload existing *.cipd files')
1564 parser.add_argument(
Vadim Shtayurad743c4c2024-11-13 19:00:51[diff] [blame]1565 '--go-bootstrap-script',
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1566 metavar='PATH',
Vadim Shtayurad743c4c2024-11-13 19:00:51[diff] [blame]1567 default=os.path.join(ROOT, 'go', 'bootstrap.py'),
1568 help='a script to use to bootstrap Go environment',
Robert Iannucci566e6072023-04-11 22:06:40[diff] [blame]1569 )
1570 parser.add_argument(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1571 '--package-definition-dir',
1572 metavar='PATH',
1573 default=os.path.join(ROOT, 'build', 'packages'),
1574 help=('points at either infra.git/build/packages or '
1575 'infra_internal.git/build/packages.'),
Robert Iannucci566e6072023-04-11 22:06:40[diff] [blame]1576 )
1577 parser.add_argument(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1578 '--package-out-dir',
1579 metavar='PATH',
1580 default=os.path.join(ROOT, 'build', 'out'),
1581 help=('points at either infra.git/build/out or '
1582 'infra_internal.git/build/out.'),
Robert Iannucci566e6072023-04-11 22:06:40[diff] [blame]1583 )
1584 parser.add_argument(
Robert Iannucci6d972442023-04-12 02:28:36[diff] [blame]1585 '--map-go-module', metavar='MOD=PATH', action='append',
1586 help='go package prefix = directory containing go.mod.',
1587 )
1588 parser.add_argument(
vadimsh72302362015-05-07 17:42:43[diff] [blame]1589 '--service-url', metavar='URL', dest='service_url',
1590 default=PACKAGE_REPO_SERVICE,
1591 help='URL of the package repository service to use')
1592 parser.add_argument(
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1593 '--service-account-json', metavar='PATH', dest='service_account_json',
1594 help='path to credentials for service account to use')
vadimshd3001422015-05-05 00:09:02[diff] [blame]1595 parser.add_argument(
1596 '--json-output', metavar='PATH', dest='json_output',
1597 help='where to dump info about built package instances')
vadimsh72302362015-05-07 17:42:43[diff] [blame]1598 parser.add_argument(
1599 '--tags', metavar='KEY:VALUE', type=str, dest='tags', nargs='*',
1600 help='tags to attach to uploaded package instances')
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1601 args = parser.parse_args(args)
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1602 if not args.build and not args.upload:
1603 parser.error('--no-rebuild doesn\'t make sense without --upload')
Robert Iannucci6d972442023-04-12 02:28:36[diff] [blame]1604
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1605 module_map = DEFAULT_MODULE_MAP.copy()
Robert Iannucci6d972442023-04-12 02:28:36[diff] [blame]1606 for mapping in (args.map_go_module or ()):
1607 mod, path = mapping.split('=')
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1608 module_map[mod] = path
Robert Iannucci6d972442023-04-12 02:28:36[diff] [blame]1609
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1610 return run(
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1611 args.cipd_platform,
Vadim Shtayurad743c4c2024-11-13 19:00:51[diff] [blame]1612 args.go_bootstrap_script,
Vadim Shtayura7e9f6982024-10-23 17:49:56[diff] [blame]1613 module_map,
Robert Iannucci566e6072023-04-11 22:06:40[diff] [blame]1614 args.package_definition_dir,
1615 args.package_out_dir,
vadimshf1072a12015-06-18 21:36:13[diff] [blame]1616 [n + '.yaml' if not n.endswith('.yaml') else n for n in args.yamls],
vadimshae9dde02015-06-04 23:52:37[diff] [blame]1617 args.build,
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1618 args.upload,
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1619 args.sign_id,
vadimsh72302362015-05-07 17:42:43[diff] [blame]1620 args.service_url,
vadimsh211f51e2015-05-07 17:57:52[diff] [blame]1621 args.tags or [],
vadimshd3001422015-05-05 00:09:02[diff] [blame]1622 args.service_account_json,
Robert Iannuccif4c04c42020-09-08 23:38:53[diff] [blame]1623 args.json_output,
Chenlin Fanf9ac0b52021-12-03 03:37:39[diff] [blame]1624 )
vadimshb545ea02015-03-11 23:08:33[diff] [blame]1625
1626
1627if __name__ == '__main__':
1628 sys.exit(main(sys.argv[1:]))