From c7ad41037ec30dd68441ec99b0804df78524a6e4 Mon Sep 17 00:00:00 2001 From: Dan Allen Date: Fri, 19 Jul 2024 01:16:14 -0600 Subject: [PATCH] resolves #3 allow target bucket to be specified using into key on scan --- CHANGELOG.adoc | 7 + .../ROOT/pages/configuration-keys.adoc | 79 ++++++++--- docs/modules/ROOT/pages/index.adoc | 1 + packages/collector-extension/lib/index.js | 50 +++++-- .../test/collector-extension-test.js | 132 +++++++++++++++++- .../.gen-component-desc-without-identity.js | 9 ++ .../test-at-root/.gen-component-desc.js | 1 + 7 files changed, 251 insertions(+), 28 deletions(-) create mode 100644 packages/collector-extension/test/fixtures/test-at-root/.gen-component-desc-without-identity.js diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 91d79c0..df0d7bb 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -4,6 +4,13 @@ This is a summary of all notable changes to the Antora Collector Extension by release. For a detailed view of what's changed, refer to the {url-repo}/commits[commit history] of this project. +== Unreleased + +=== Added + +* allow target bucket to be specified using `into` key on scan operation (#3) +* allow different target bucket to be specified in scanned antora.yml when `create` key is true (#3) + == 1.0.0-alpha.6 (2024-07-17) === Fixed diff --git a/docs/modules/ROOT/pages/configuration-keys.adoc b/docs/modules/ROOT/pages/configuration-keys.adoc index 8fce94a..454f0d2 100644 --- a/docs/modules/ROOT/pages/configuration-keys.adoc +++ b/docs/modules/ROOT/pages/configuration-keys.adoc @@ -44,7 +44,8 @@ ext: The `collector` key accepts an array (i.e., list) of collector entries. These entries can configure any combination of clean, run, and scan operations. -If there's only a single entry, the array can be replaced by a map for a single entry (i.e., no preceding hyphen). + +TIP: If there's only a single entry, the array can be replaced by a map (i.e., drop the leading `-` marker). If the `collector` key isn't set in a component version descriptor, or its value is falsy, the extension won't run on that git reference. @@ -52,7 +53,7 @@ If the `collector` key isn't set in a component version descriptor, or its value == clean key The `clean` key must be nested under the `collector` key. -If the `collector` key is an array, the `clean` key must be specified as a key on one of its entries. +If the `collector` key is an array, the `clean` key must be specified as a key on an array entry. .antora.yml [,yaml] @@ -77,7 +78,8 @@ ext: The `clean` key accepts an array of items (i.e., list). An entry in the `clean` key is a map with a single key, `dir`, which specifies the directory to clean. Each `clean` entry is invoked sequentially in the order specified in the array. -If there's only a single entry, the array can be replaced by a map for a single entry. + +TIP: If there's only a single entry, the array can be replaced by a map (i.e., drop the leading `-` marker). If the directory to clean is the same as the directory to scan, the clean entry can be created implicitly by setting the `clean` key on the scan entry to `true` (e.g., `clean: true`). See <>. @@ -95,7 +97,7 @@ In this case, the first clean entry can be important to ensuring a predictable r == run key The `run` key is nested under the `collector` key. -If the `collector` key is an array, the `run` key must be specified as a key on one of its entries. +If the `collector` key is an array, the `run` key must be specified as a key on an array entry. .antora.yml [,yaml] @@ -121,7 +123,8 @@ ext: The `run` key accepts an array of items (i.e., list). An entry in the `run` key is a map of built-in key-value pairs that configure the run, which includes the directory (`dir`) and command string (`command`). Each `run` entry is invoked sequentially in the order specified in the array. -If there's only a single entry, the array can be replaced by a map for a single entry. + +TIP: If there's only a single entry, the array can be replaced by a map (i.e., drop the leading `-` marker). If the `local` key is specified with the value `true`, the command must be located relative to where the command is run. This key can be set implicitly by prepending the command with `./`. @@ -140,7 +143,7 @@ If the value is `throw` (the default), the error bubbles, which causes the execu == scan key The `scan` key must be nested under the `collector` key. -If the `collector` key is an array, the `scan` key must be specified as a key on one of its entries. +If the `collector` key is an array, the `scan` key must be specified as a key on an array entry. .antora.yml [,yaml] @@ -165,14 +168,17 @@ ext: The `scan` key accepts an array of items (i.e., list). An entry in the `scan` key is a map of built-in key-value pairs that configure the scan, which includes the directory (`dir`) and file filter (`files`). Each `scan` entry is invoked sequentially in the order specified in the array. -If there's only a single entry, the array can be replaced by a map for a single entry. -If the `clean: true` key is set on the entry, that implicitly creates a clean entry with the same dir. +TIP: If there's only a single entry, the array can be replaced by a map (i.e., drop the leading `-` marker). + +If the `clean: true` key is set on the entry, that implicitly creates a separate clean entry using the same dir. -For each file discovered by a scan, an virtual file entry is added to the entry in the content aggregate for the current component and version. -The file is added using the path relative to the scan dir (e.g., [.path]_modules/ROOT/pages/generated.adoc_). +For each file discovered by a scan, a virtual file is added to the bucket in the content aggregate for the current component and version (as specified in [.path]_antora.yml_) by default. +The file is added using the path relative to the scan dir. +Thus, there's an assumption that the scanned files are organized according to the standard Antora structure (e.g., [.path]_modules/ROOT/pages/generated.adoc_). -If the `into` key is specified, that value is prepended to the relative path of the file. +If the files are not organized as an Antora content root, they can be remapped using the scan configuration. +If the `into` key is specified, that value is prepended as a base directory path to the relative path of every scanned file. For example: [,yaml] @@ -182,11 +188,51 @@ scan: into: modules/ROOT/pages ---- -By using the `into` key, the scanned files do not have to be organized in the standard Antora structure. +By using the `into` key, the scanned files do not have to be organized according to the standard Antora structure. + +The `into` key also provides a way to import the files into a component version bucket that's different from the one in which Collector is running. +To do so, you can specify the `name` and/or `version` key on the `into` key. +(If only one of the keys is specified, the value of the other key inherits from the current component version descriptor). +For example: + +[,yaml] +---- +scan: + dir: build/apidocs + into: + name: apidocs + version: ~ +---- -The files discovered by a scan operation are added to the content aggregate *before* the content is classified. -Thus, these files become indistinguishable from ones that are discovered in a content root. -It's as though the discovered files are the repository branch that Antora scans, only they are added dynamically after the fact. +The same can be achieved by specifying the target name and version in a scanned [.path]_antora.yml_ file and also setting the `create: true` key. +For example: + +.antora.yml +[,yaml] +---- +name: apidocs +version: ~ +create: true +---- + +Setting the `create: true` key tells Collector to use the target bucket, creating it if necessary, instead of updating the identity of the current bucket. + +If you want to prepend a base directory path to the files while also specifying a target component version bucket, the path should be specified in the `dir` key. +For example: + +[,yaml] +---- +scan: + dir: build/apidocs + into: + name: apidocs + version: ~ + dir: modules/ROOT/attachments +---- + +The files discovered by a scan operation are added to the component version bucket in the content aggregate *before* the content is classified. +Thus, these files are indistinguishable from ones discovered in a static Antora content root. +It's as though the discovered files are in the repository branch that Antora scans, only they are added dynamically later. [#collector-reference] == Available Collector keys @@ -253,8 +299,9 @@ The `files` key accepts a micromatch pattern (supporting the same syntax as the |`scan.into` |Specifies a path to prepend to the relative path of all files discovered by the current scan operation. This path should also be relative (e.g., [.path]_modules/ROOT/pages_) +When the value is a map, can also be used to specify the target component name and/or version. |Not set -|Path relative to the content root +|Path relative to the content root or a map with the optional keys `name`, `version`, and `dir`. |`scan.clean` |Specifies that the scanned path should be cleaned before processing this collector entry. diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index ea7a1ef..eb063af 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -28,6 +28,7 @@ Those files, in turn, get added to the content catalog. The files that Collector discovers are imported into the content aggregate bucket for the component version in which Collector is configured to run. By default, Collector assumes the files that it scans are organized in the standard Antora structure of a content root. It's possible to graf the scanned files onto a specified path prefix using the `into` key (e.g., [.path]_modules/ROOT/examples_). +The `into` key can also be used to import the scanned files into a different component version bucket. The purpose of Collector is to contribute additional files to a content source root. You can think of Collector as adding additional content roots to your playbook that may happen to be created on the fly (and transient). diff --git a/packages/collector-extension/lib/index.js b/packages/collector-extension/lib/index.js index 296808d..0a85ac6 100644 --- a/packages/collector-extension/lib/index.js +++ b/packages/collector-extension/lib/index.js @@ -27,8 +27,8 @@ module.exports.register = function ({ config: { keepWorktrees = false } }) { await fsp.mkdir(cacheDir, { recursive: true }) const gitCache = {} const managedWorktrees = new Map() - for (const componentVersionBucket of contentAggregate) { - const { files: filesInBucket, origins = [] } = componentVersionBucket + for (const componentVersionBucket of contentAggregate.slice()) { + const { origins = [] } = componentVersionBucket for (const origin of origins) { const { url, gitdir, refname, reftype, remote, worktree, startPath, descriptor } = origin let collectorConfig = descriptor?.ext?.collector || [] @@ -108,21 +108,51 @@ module.exports.register = function ({ config: { keepWorktrees = false } }) { } } for (const scan of scans) { - for (const file of await srcFs(scan.dir, scan.files, scan.into ?? scan.base)) { - let existingFile + let into = ('into' in scan ? scan.into : scan.base) || {} + if (typeof into === 'string') into = { dir: into } + let componentVersionDesc + const files = await srcFs(scan.dir, scan.files, into.dir).then((result) => + result.filter((file) => (file.path === 'antora.yml' ? !(componentVersionDesc = file) : true)) + ) + let requestedBucketName = 'name' in into ? String(into.name) : componentVersionBucket.name + let requestedBucketVersion = 'version' in into ? String(into.version || '') : componentVersionBucket.version + let replaceTargetBucket + if ((componentVersionDesc &&= yaml.load(componentVersionDesc.contents))) { + if ('name' in componentVersionDesc) requestedBucketName = String(componentVersionDesc.name) + if ('version' in componentVersionDesc) requestedBucketVersion = String(componentVersionDesc.version || '') + replaceTargetBucket = !componentVersionDesc.create + delete componentVersionDesc.create + Object.assign(componentVersionDesc, { name: requestedBucketName, version: requestedBucketVersion }) + } + let targetBucket = componentVersionBucket + if ( + !replaceTargetBucket && + !(targetBucket.name === requestedBucketName && targetBucket.version === requestedBucketVersion) + ) { + targetBucket = contentAggregate.find( + (candidate) => candidate.name === requestedBucketName && candidate.version === requestedBucketVersion + ) + if (!targetBucket) { + targetBucket = { name: requestedBucketName, version: requestedBucketVersion, files: [], origins: [] } + contentAggregate.push(targetBucket) + } + } + if (componentVersionDesc) { + Object.assign(targetBucket, componentVersionDesc) + if (replaceTargetBucket && !('prerelease' in componentVersionDesc)) delete targetBucket.prerelease + } + const targetFiles = targetBucket.files + for (const file of files) { const relpath = file.path - if (relpath === 'antora.yml') { - const generated = yaml.load(file.contents) - Object.assign(componentVersionBucket, generated) - if (!('prerelease' in generated)) delete componentVersionBucket.prerelease - } else if ((existingFile = filesInBucket.find(({ path: path_ }) => path_ === relpath))) { + const existingFile = targetFiles.find((it) => it.path === relpath) + if (existingFile) { Object.assign(existingFile, { contents: file.contents, stat: file.stat }) } else { const src = file.src const scannedRelpath = src.abspath.substr(worktreeDir.length + 1) Object.assign(src, { origin, scanned: posixify ? posixify(scannedRelpath) : scannedRelpath }) if (!worktree) Object.assign(src, { realpath: src.abspath, abspath: src.scanned }) - filesInBucket.push(file) + targetFiles.push(file) } } } diff --git a/packages/collector-extension/test/collector-extension-test.js b/packages/collector-extension/test/collector-extension-test.js index 7ec7b3f..0596e10 100644 --- a/packages/collector-extension/test/collector-extension-test.js +++ b/packages/collector-extension/test/collector-extension-test.js @@ -411,6 +411,31 @@ describe('collector extension', () => { }) }) + it('should inherit name and version if not specified antora.yml', async () => { + const collectorConfig = { + run: { command: 'node .gen-component-desc-without-identity.js' }, + scan: { dir: 'build' }, + } + let bucketBefore + await runScenario({ + repoName: 'test-at-root', + collectorConfig, + before: (contentAggregate) => { + expect(contentAggregate).to.have.lengthOf(1) + bucketBefore = contentAggregate[0] + expect(bucketBefore.files).to.be.empty() + }, + after: (contentAggregate) => { + expect(contentAggregate).to.have.lengthOf(1) + const bucketAfter = contentAggregate[0] + expect(bucketAfter.name).to.equal(bucketBefore.name) + expect(bucketAfter.version).to.equal(bucketBefore.version) + expect(bucketAfter).to.have.nested.property('asciidoc.attributes.url-api', 'https://api.example.org') + expect(bucketAfter.files).to.be.empty() + }, + }) + }) + it('should only update contents of existing page', async () => { const collectorConfig = { run: { command: 'node .gen-replace-start-page.js' }, @@ -635,7 +660,7 @@ describe('collector extension', () => { }) }) - it('should rebase files if base key is set on scan', async () => { + it('should rebase files if into key is set on scan', async () => { const collectorConfig = { scan: { dir: 'code', into: 'modules/extend/examples' }, } @@ -654,7 +679,7 @@ describe('collector extension', () => { }) }) - it('should drop leading / on value of base key', async () => { + it('should drop leading / on value of into key', async () => { const collectorConfig = { scan: { dir: 'code', into: '/modules/extend/examples' }, } @@ -673,6 +698,109 @@ describe('collector extension', () => { }) }) + it('should put files into bucket specified on scan', async () => { + const collectorConfig = { + scan: { dir: 'code', into: { name: 'shared', version: '', dir: 'modules/ROOT/examples' } }, + } + await runScenario({ + repoName: 'test-at-start-path', + startPath: 'docs', + collectorConfig, + before: (contentAggregate) => { + expect(contentAggregate).to.have.lengthOf(1) + expect(contentAggregate[0].files).to.be.empty() + }, + after: (contentAggregate) => { + expect(contentAggregate).to.have.lengthOf(2) + expect(contentAggregate[1].files).to.have.lengthOf(1) + expect(contentAggregate[1].name).to.equal('shared') + expect(contentAggregate[1].version).to.equal('') + expect(contentAggregate[1].files[0].path).to.equal('modules/ROOT/examples/extended-pdf-converter.rb') + }, + }) + }) + + it('should put files into existing bucket specified in antora.yml', async () => { + const collectorConfig = { + run: [{ command: 'node .gen-component-desc.js create' }, { command: 'node .gen-more-files.js' }], + scan: { dir: 'build' }, + } + await runScenario({ + repoName: 'test-at-root', + collectorConfig, + before: (contentAggregate) => { + expect(contentAggregate).to.have.lengthOf(1) + const bucket = contentAggregate[0] + expect(bucket.version).to.equal('main') + expect(bucket.files).to.be.empty() + contentAggregate.push({ + name: 'test', + version: '1.0.0', + files: [], + origins: [], + }) + }, + after: (contentAggregate) => { + expect(contentAggregate).to.have.lengthOf(2) + const bucket = contentAggregate[1] + expect(bucket.files).to.have.lengthOf(1) + expect(bucket.name).to.equal('test') + expect(bucket.version).to.equal('1.0.0') + expect(bucket.files[0].path).to.equal('modules/more/pages/index.adoc') + }, + }) + }) + + it('should put files into new bucket specified in antora.yml', async () => { + const collectorConfig = { + run: [{ command: 'node .gen-component-desc.js create' }, { command: 'node .gen-more-files.js' }], + scan: { dir: 'build' }, + } + await runScenario({ + repoName: 'test-at-root', + collectorConfig, + before: (contentAggregate) => { + expect(contentAggregate).to.have.lengthOf(1) + const bucket = contentAggregate[0] + expect(bucket.version).to.equal('main') + expect(bucket.files).to.be.empty() + }, + after: (contentAggregate) => { + expect(contentAggregate).to.have.lengthOf(2) + const bucket = contentAggregate[1] + expect(bucket.files).to.have.lengthOf(1) + expect(bucket.name).to.equal('test') + expect(bucket.version).to.equal('1.0.0') + expect(bucket.files[0].path).to.equal('modules/more/pages/index.adoc') + }, + }) + }) + + it('should allow bucket in antora.yml to override bucket specified on scan', async () => { + const collectorConfig = { + run: [{ command: 'node .gen-component-desc.js' }, { command: 'node .gen-more-files.js' }], + scan: { dir: 'build', into: { name: 'overridden-name', version: 'overridden-version' } }, + } + await runScenario({ + repoName: 'test-at-root', + collectorConfig, + before: (contentAggregate) => { + expect(contentAggregate).to.have.lengthOf(1) + const bucket = contentAggregate[0] + expect(bucket.version).to.equal('main') + expect(bucket.files).to.be.empty() + }, + after: (contentAggregate) => { + expect(contentAggregate).to.have.lengthOf(1) + const bucket = contentAggregate[0] + expect(bucket.files).to.have.lengthOf(1) + expect(bucket.name).to.equal('test') + expect(bucket.version).to.equal('1.0.0') + expect(bucket.files[0].path).to.equal('modules/more/pages/index.adoc') + }, + }) + }) + it('should collect files from multiple scan dirs', async () => { const collectorConfig = { run: { command: 'node .gen-start-page.js', dir: '.' }, diff --git a/packages/collector-extension/test/fixtures/test-at-root/.gen-component-desc-without-identity.js b/packages/collector-extension/test/fixtures/test-at-root/.gen-component-desc-without-identity.js new file mode 100644 index 0000000..4e2211a --- /dev/null +++ b/packages/collector-extension/test/fixtures/test-at-root/.gen-component-desc-without-identity.js @@ -0,0 +1,9 @@ +'use strict' + +const fsp = require('node:fs/promises') + +;(async () => { + await fsp.mkdir('build', { recursive: true }) + const antoraYml = 'title: Test\n' + 'asciidoc:\n' + ' attributes:\n' + ' url-api: https://api.example.org\n' + await fsp.writeFile('build/antora.yml', antoraYml, 'utf8') +})() diff --git a/packages/collector-extension/test/fixtures/test-at-root/.gen-component-desc.js b/packages/collector-extension/test/fixtures/test-at-root/.gen-component-desc.js index 966e16f..e18f30c 100644 --- a/packages/collector-extension/test/fixtures/test-at-root/.gen-component-desc.js +++ b/packages/collector-extension/test/fixtures/test-at-root/.gen-component-desc.js @@ -7,6 +7,7 @@ const fsp = require('node:fs/promises') const antoraYml = 'name: test\n' + 'version: 1.0.0\n' + + (process.argv[2] === 'create' ? 'create: true\n' : '') + 'title: Test\n' + 'asciidoc:\n' + ' attributes:\n' + -- GitLab