Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions src/buildstream/_frontend/cli.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -1694,3 +1694,124 @@ def artifact_delete(app, artifacts, deps):
"""Remove artifacts from the local cache"""
with app.initialized():
app.stream.artifact_delete(artifacts, selection=deps)


#############################################################
# Buildtree Commands #
#############################################################
@cli.group(short_help="Manipulate cached buildtree.")
def buildtree():
"""Manipulate cached buildtree"""


#####################################################################
# Buildtree Checkout Command #
#####################################################################
@buildtree.command(name="checkout", short_help="Checkout contents of a buildtree")
@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.")
@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible")
@click.option(
"--tar",
default=None,
metavar="LOCATION",
type=click.Path(),
help="Create a tarball from the artifact contents instead "
"of a file tree. If LOCATION is '-', the tarball "
"will be dumped to the standard output.",
)
@click.option(
"--compression",
default=None,
type=click.Choice(["gz", "xz", "bz2"]),
help="The compression option of the tarball created.",
)
@click.option(
"--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to"
)
@click.option(
"--artifact-remote",
"artifact_remotes",
type=RemoteSpecType(RemoteSpecPurpose.PULL),
multiple=True,
help="A remote for downloading artifacts",
)
@click.option(
"--ignore-project-artifact-remotes",
is_flag=True,
help="Ignore remote artifact cache servers recommended by projects",
)
@click.argument("target", required=False, type=click.Path(readable=False))
@click.pass_obj
def buildtree_checkout(
app,
buildroot,
force,
hardlinks,
tar,
compression,
directory,
artifact_remotes,
ignore_project_artifact_remotes,
target,
):
"""Checkout buildtree

When this command is executed from a workspace directory, the default
is to checkout the artifact of the workspace element.
"""
from .. import utils

if hardlinks and tar:
click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
sys.exit(-1)

if tar and directory:
click.echo("ERROR: options --directory and --tar conflict", err=True)
sys.exit(-1)

if not tar:
if compression:
click.echo("ERROR: --compression can only be provided if --tar is provided", err=True)
sys.exit(-1)
else:
location = tar
try:
inferred_compression = utils._get_compression(tar)
except UtilError as e:
click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True)
sys.exit(-1)
if compression and inferred_compression != "" and inferred_compression != compression:
click.echo(
"WARNING: File extension and compression differ."
"File extension has been overridden by --compression",
err=True,
)
if not compression:
compression = inferred_compression

with app.initialized():
if not target:
target = app.stream.get_default_target()
if not target:
raise AppError('Missing argument "ELEMENT".')

if not tar:
if directory is None:
location = os.path.abspath(os.path.join(os.getcwd(), target))
if location[-4:] == ".bst":
location = location[:-4]
else:
location = directory

app.stream.buildtree_checkout(
target,
location=location,
buildroot=buildroot,
force=force,
hardlinks=hardlinks,
compression=compression,
tar=bool(tar),
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)
78 changes: 78 additions & 0 deletions src/buildstream/_stream.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -916,6 +916,84 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE):
if not ref_removed:
self._context.messenger.info("No artifacts were removed")

# buildtree_checkout()
#
# Checkout target buildtree artifact to the specified location
#
# Args:
# target: Target to checkout
# location: Location to checkout the artifact to
# force: Whether files can be overwritten if necessary
# hardlinks: Whether checking out files hardlinked to
# their artifacts is acceptable
# tar: If true, a tarball from the artifact contents will
# be created, otherwise the file tree of the artifact
# will be placed at the given location. If true and
# location is '-', the tarball will be dumped on the
# standard output.
# artifact_remotes: Artifact cache remotes specified on the commmand line
# ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects
#
def buildtree_checkout(
self,
target: str,
*,
location: Optional[str] = None,
buildroot: bool = False,
force: bool = False,
hardlinks: bool = False,
compression: str = "",
tar: bool = False,
artifact_remotes: Iterable[RemoteSpec] = (),
ignore_project_artifact_remotes: bool = False,
):
elements = self._load(
(target,),
selection=_PipelineSelection.NONE,
load_artifacts=True,
attempt_artifact_metadata=True,
connect_artifact_cache=True,
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)

assert len(elements) == 1
element: Element = elements[0]

self._check_location_writable(location, force=force, tar=tar)

self.query_cache([element])
self._pull_missing_artifacts([element])

if buildroot:
# If the user requested the buildroot but no buildroot is available, fail.
if not element._buildroot_exists():
raise StreamError("No buildroot artifact available", reason="missing-buildroot")
else:
# If the user requested the buildtree but no buildtree is available,
# suggest checking out the buildroot if one is available.
# Otherwise, fail if neither is available.
if not element._buildtree_exists():
if element._buildroot_exists():
raise StreamError(
"No buildtree artifact available, but a buildroot is available. "
"Use the --buildroot option to check out the buildroot.",
reason="missing-buildtree-buildroot-exists",
)
raise StreamError(
"Neither a buildtree nor a buildroot artifact is available.",
reason="missing-buildtree-and-buildroot",
)

try:
artifact = element._get_artifact()
virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another thing we need to be aware of (and also give a suggestion about to the user) is that not all elements have a build tree: only build elements do.

So it's not enough to check for the build root, we also need to check for the build tree. Then depending on the availability and the user request, we can suggest an alternative to the user.

buildtree availablebuildroot availableuser requestedresult
YYbuild treeextract build tree
YYbuild rootextract build root
NYbuild treeerror + suggest extract build root
NYbuild rootextract build root
NNbuild treeerror
NNbuild rooterror

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for pointing this out. I've accounted for all of the cases you described and added tests covering them.

Could you please take another look when you have a chance?

self._export_artifact(tar, location, compression, element, hardlinks, virdir)
except BstError as e:
raise StreamError(
"Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason
) from e

# source_checkout()
#
# Checkout sources of the target element to the specified location
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all \u003cpre\u003e\u003ccode\u003e blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks"); } } catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); } })(); (function(){ try { var __m = "github.com"; var __re = new RegExp('^' + "github\\.com" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions src/buildstream/_frontend/cli.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -1694,3 +1694,124 @@ def artifact_delete(app, artifacts, deps):
"""Remove artifacts from the local cache"""
with app.initialized():
app.stream.artifact_delete(artifacts, selection=deps)


#############################################################
# Buildtree Commands #
#############################################################
@cli.group(short_help="Manipulate cached buildtree.")
def buildtree():
"""Manipulate cached buildtree"""


#####################################################################
# Buildtree Checkout Command #
#####################################################################
@buildtree.command(name="checkout", short_help="Checkout contents of a buildtree")
@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.")
@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible")
@click.option(
"--tar",
default=None,
metavar="LOCATION",
type=click.Path(),
help="Create a tarball from the artifact contents instead "
"of a file tree. If LOCATION is '-', the tarball "
"will be dumped to the standard output.",
)
@click.option(
"--compression",
default=None,
type=click.Choice(["gz", "xz", "bz2"]),
help="The compression option of the tarball created.",
)
@click.option(
"--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to"
)
@click.option(
"--artifact-remote",
"artifact_remotes",
type=RemoteSpecType(RemoteSpecPurpose.PULL),
multiple=True,
help="A remote for downloading artifacts",
)
@click.option(
"--ignore-project-artifact-remotes",
is_flag=True,
help="Ignore remote artifact cache servers recommended by projects",
)
@click.argument("target", required=False, type=click.Path(readable=False))
@click.pass_obj
def buildtree_checkout(
app,
buildroot,
force,
hardlinks,
tar,
compression,
directory,
artifact_remotes,
ignore_project_artifact_remotes,
target,
):
"""Checkout buildtree

When this command is executed from a workspace directory, the default
is to checkout the artifact of the workspace element.
"""
from .. import utils

if hardlinks and tar:
click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
sys.exit(-1)

if tar and directory:
click.echo("ERROR: options --directory and --tar conflict", err=True)
sys.exit(-1)

if not tar:
if compression:
click.echo("ERROR: --compression can only be provided if --tar is provided", err=True)
sys.exit(-1)
else:
location = tar
try:
inferred_compression = utils._get_compression(tar)
except UtilError as e:
click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True)
sys.exit(-1)
if compression and inferred_compression != "" and inferred_compression != compression:
click.echo(
"WARNING: File extension and compression differ."
"File extension has been overridden by --compression",
err=True,
)
if not compression:
compression = inferred_compression

with app.initialized():
if not target:
target = app.stream.get_default_target()
if not target:
raise AppError('Missing argument "ELEMENT".')

if not tar:
if directory is None:
location = os.path.abspath(os.path.join(os.getcwd(), target))
if location[-4:] == ".bst":
location = location[:-4]
else:
location = directory

app.stream.buildtree_checkout(
target,
location=location,
buildroot=buildroot,
force=force,
hardlinks=hardlinks,
compression=compression,
tar=bool(tar),
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)
78 changes: 78 additions & 0 deletions src/buildstream/_stream.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -916,6 +916,84 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE):
if not ref_removed:
self._context.messenger.info("No artifacts were removed")

# buildtree_checkout()
#
# Checkout target buildtree artifact to the specified location
#
# Args:
# target: Target to checkout
# location: Location to checkout the artifact to
# force: Whether files can be overwritten if necessary
# hardlinks: Whether checking out files hardlinked to
# their artifacts is acceptable
# tar: If true, a tarball from the artifact contents will
# be created, otherwise the file tree of the artifact
# will be placed at the given location. If true and
# location is '-', the tarball will be dumped on the
# standard output.
# artifact_remotes: Artifact cache remotes specified on the commmand line
# ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects
#
def buildtree_checkout(
self,
target: str,
*,
location: Optional[str] = None,
buildroot: bool = False,
force: bool = False,
hardlinks: bool = False,
compression: str = "",
tar: bool = False,
artifact_remotes: Iterable[RemoteSpec] = (),
ignore_project_artifact_remotes: bool = False,
):
elements = self._load(
(target,),
selection=_PipelineSelection.NONE,
load_artifacts=True,
attempt_artifact_metadata=True,
connect_artifact_cache=True,
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)

assert len(elements) == 1
element: Element = elements[0]

self._check_location_writable(location, force=force, tar=tar)

self.query_cache([element])
self._pull_missing_artifacts([element])

if buildroot:
# If the user requested the buildroot but no buildroot is available, fail.
if not element._buildroot_exists():
raise StreamError("No buildroot artifact available", reason="missing-buildroot")
else:
# If the user requested the buildtree but no buildtree is available,
# suggest checking out the buildroot if one is available.
# Otherwise, fail if neither is available.
if not element._buildtree_exists():
if element._buildroot_exists():
raise StreamError(
"No buildtree artifact available, but a buildroot is available. "
"Use the --buildroot option to check out the buildroot.",
reason="missing-buildtree-buildroot-exists",
)
raise StreamError(
"Neither a buildtree nor a buildroot artifact is available.",
reason="missing-buildtree-and-buildroot",
)

try:
artifact = element._get_artifact()
virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another thing we need to be aware of (and also give a suggestion about to the user) is that not all elements have a build tree: only build elements do.

So it's not enough to check for the build root, we also need to check for the build tree. Then depending on the availability and the user request, we can suggest an alternative to the user.

buildtree availablebuildroot availableuser requestedresult
YYbuild treeextract build tree
YYbuild rootextract build root
NYbuild treeerror + suggest extract build root
NYbuild rootextract build root
NNbuild treeerror
NNbuild rooterror

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for pointing this out. I've accounted for all of the cases you described and added tests covering them.

Could you please take another look when you have a chance?

self._export_artifact(tar, location, compression, element, hardlinks, virdir)
except BstError as e:
raise StreamError(
"Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason
) from e

# source_checkout()
#
# Checkout sources of the target element to the specified location
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions src/buildstream/_frontend/cli.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -1694,3 +1694,124 @@ def artifact_delete(app, artifacts, deps):
"""Remove artifacts from the local cache"""
with app.initialized():
app.stream.artifact_delete(artifacts, selection=deps)


#############################################################
# Buildtree Commands #
#############################################################
@cli.group(short_help="Manipulate cached buildtree.")
def buildtree():
"""Manipulate cached buildtree"""


#####################################################################
# Buildtree Checkout Command #
#####################################################################
@buildtree.command(name="checkout", short_help="Checkout contents of a buildtree")
@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.")
@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible")
@click.option(
"--tar",
default=None,
metavar="LOCATION",
type=click.Path(),
help="Create a tarball from the artifact contents instead "
"of a file tree. If LOCATION is '-', the tarball "
"will be dumped to the standard output.",
)
@click.option(
"--compression",
default=None,
type=click.Choice(["gz", "xz", "bz2"]),
help="The compression option of the tarball created.",
)
@click.option(
"--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to"
)
@click.option(
"--artifact-remote",
"artifact_remotes",
type=RemoteSpecType(RemoteSpecPurpose.PULL),
multiple=True,
help="A remote for downloading artifacts",
)
@click.option(
"--ignore-project-artifact-remotes",
is_flag=True,
help="Ignore remote artifact cache servers recommended by projects",
)
@click.argument("target", required=False, type=click.Path(readable=False))
@click.pass_obj
def buildtree_checkout(
app,
buildroot,
force,
hardlinks,
tar,
compression,
directory,
artifact_remotes,
ignore_project_artifact_remotes,
target,
):
"""Checkout buildtree

When this command is executed from a workspace directory, the default
is to checkout the artifact of the workspace element.
"""
from .. import utils

if hardlinks and tar:
click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
sys.exit(-1)

if tar and directory:
click.echo("ERROR: options --directory and --tar conflict", err=True)
sys.exit(-1)

if not tar:
if compression:
click.echo("ERROR: --compression can only be provided if --tar is provided", err=True)
sys.exit(-1)
else:
location = tar
try:
inferred_compression = utils._get_compression(tar)
except UtilError as e:
click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True)
sys.exit(-1)
if compression and inferred_compression != "" and inferred_compression != compression:
click.echo(
"WARNING: File extension and compression differ."
"File extension has been overridden by --compression",
err=True,
)
if not compression:
compression = inferred_compression

with app.initialized():
if not target:
target = app.stream.get_default_target()
if not target:
raise AppError('Missing argument "ELEMENT".')

if not tar:
if directory is None:
location = os.path.abspath(os.path.join(os.getcwd(), target))
if location[-4:] == ".bst":
location = location[:-4]
else:
location = directory

app.stream.buildtree_checkout(
target,
location=location,
buildroot=buildroot,
force=force,
hardlinks=hardlinks,
compression=compression,
tar=bool(tar),
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)
78 changes: 78 additions & 0 deletions src/buildstream/_stream.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -916,6 +916,84 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE):
if not ref_removed:
self._context.messenger.info("No artifacts were removed")

# buildtree_checkout()
#
# Checkout target buildtree artifact to the specified location
#
# Args:
# target: Target to checkout
# location: Location to checkout the artifact to
# force: Whether files can be overwritten if necessary
# hardlinks: Whether checking out files hardlinked to
# their artifacts is acceptable
# tar: If true, a tarball from the artifact contents will
# be created, otherwise the file tree of the artifact
# will be placed at the given location. If true and
# location is '-', the tarball will be dumped on the
# standard output.
# artifact_remotes: Artifact cache remotes specified on the commmand line
# ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects
#
def buildtree_checkout(
self,
target: str,
*,
location: Optional[str] = None,
buildroot: bool = False,
force: bool = False,
hardlinks: bool = False,
compression: str = "",
tar: bool = False,
artifact_remotes: Iterable[RemoteSpec] = (),
ignore_project_artifact_remotes: bool = False,
):
elements = self._load(
(target,),
selection=_PipelineSelection.NONE,
load_artifacts=True,
attempt_artifact_metadata=True,
connect_artifact_cache=True,
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)

assert len(elements) == 1
element: Element = elements[0]

self._check_location_writable(location, force=force, tar=tar)

self.query_cache([element])
self._pull_missing_artifacts([element])

if buildroot:
# If the user requested the buildroot but no buildroot is available, fail.
if not element._buildroot_exists():
raise StreamError("No buildroot artifact available", reason="missing-buildroot")
else:
# If the user requested the buildtree but no buildtree is available,
# suggest checking out the buildroot if one is available.
# Otherwise, fail if neither is available.
if not element._buildtree_exists():
if element._buildroot_exists():
raise StreamError(
"No buildtree artifact available, but a buildroot is available. "
"Use the --buildroot option to check out the buildroot.",
reason="missing-buildtree-buildroot-exists",
)
raise StreamError(
"Neither a buildtree nor a buildroot artifact is available.",
reason="missing-buildtree-and-buildroot",
)

try:
artifact = element._get_artifact()
virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another thing we need to be aware of (and also give a suggestion about to the user) is that not all elements have a build tree: only build elements do.

So it's not enough to check for the build root, we also need to check for the build tree. Then depending on the availability and the user request, we can suggest an alternative to the user.

buildtree availablebuildroot availableuser requestedresult
YYbuild treeextract build tree
YYbuild rootextract build root
NYbuild treeerror + suggest extract build root
NYbuild rootextract build root
NNbuild treeerror
NNbuild rooterror

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for pointing this out. I've accounted for all of the cases you described and added tests covering them.

Could you please take another look when you have a chance?

self._export_artifact(tar, location, compression, element, hardlinks, virdir)
except BstError as e:
raise StreamError(
"Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason
) from e

# source_checkout()
#
# Checkout sources of the target element to the specified location
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length \u003e 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions src/buildstream/_frontend/cli.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -1694,3 +1694,124 @@ def artifact_delete(app, artifacts, deps):
"""Remove artifacts from the local cache"""
with app.initialized():
app.stream.artifact_delete(artifacts, selection=deps)


#############################################################
# Buildtree Commands #
#############################################################
@cli.group(short_help="Manipulate cached buildtree.")
def buildtree():
"""Manipulate cached buildtree"""


#####################################################################
# Buildtree Checkout Command #
#####################################################################
@buildtree.command(name="checkout", short_help="Checkout contents of a buildtree")
@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.")
@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible")
@click.option(
"--tar",
default=None,
metavar="LOCATION",
type=click.Path(),
help="Create a tarball from the artifact contents instead "
"of a file tree. If LOCATION is '-', the tarball "
"will be dumped to the standard output.",
)
@click.option(
"--compression",
default=None,
type=click.Choice(["gz", "xz", "bz2"]),
help="The compression option of the tarball created.",
)
@click.option(
"--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to"
)
@click.option(
"--artifact-remote",
"artifact_remotes",
type=RemoteSpecType(RemoteSpecPurpose.PULL),
multiple=True,
help="A remote for downloading artifacts",
)
@click.option(
"--ignore-project-artifact-remotes",
is_flag=True,
help="Ignore remote artifact cache servers recommended by projects",
)
@click.argument("target", required=False, type=click.Path(readable=False))
@click.pass_obj
def buildtree_checkout(
app,
buildroot,
force,
hardlinks,
tar,
compression,
directory,
artifact_remotes,
ignore_project_artifact_remotes,
target,
):
"""Checkout buildtree

When this command is executed from a workspace directory, the default
is to checkout the artifact of the workspace element.
"""
from .. import utils

if hardlinks and tar:
click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
sys.exit(-1)

if tar and directory:
click.echo("ERROR: options --directory and --tar conflict", err=True)
sys.exit(-1)

if not tar:
if compression:
click.echo("ERROR: --compression can only be provided if --tar is provided", err=True)
sys.exit(-1)
else:
location = tar
try:
inferred_compression = utils._get_compression(tar)
except UtilError as e:
click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True)
sys.exit(-1)
if compression and inferred_compression != "" and inferred_compression != compression:
click.echo(
"WARNING: File extension and compression differ."
"File extension has been overridden by --compression",
err=True,
)
if not compression:
compression = inferred_compression

with app.initialized():
if not target:
target = app.stream.get_default_target()
if not target:
raise AppError('Missing argument "ELEMENT".')

if not tar:
if directory is None:
location = os.path.abspath(os.path.join(os.getcwd(), target))
if location[-4:] == ".bst":
location = location[:-4]
else:
location = directory

app.stream.buildtree_checkout(
target,
location=location,
buildroot=buildroot,
force=force,
hardlinks=hardlinks,
compression=compression,
tar=bool(tar),
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)
78 changes: 78 additions & 0 deletions src/buildstream/_stream.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -916,6 +916,84 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE):
if not ref_removed:
self._context.messenger.info("No artifacts were removed")

# buildtree_checkout()
#
# Checkout target buildtree artifact to the specified location
#
# Args:
# target: Target to checkout
# location: Location to checkout the artifact to
# force: Whether files can be overwritten if necessary
# hardlinks: Whether checking out files hardlinked to
# their artifacts is acceptable
# tar: If true, a tarball from the artifact contents will
# be created, otherwise the file tree of the artifact
# will be placed at the given location. If true and
# location is '-', the tarball will be dumped on the
# standard output.
# artifact_remotes: Artifact cache remotes specified on the commmand line
# ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects
#
def buildtree_checkout(
self,
target: str,
*,
location: Optional[str] = None,
buildroot: bool = False,
force: bool = False,
hardlinks: bool = False,
compression: str = "",
tar: bool = False,
artifact_remotes: Iterable[RemoteSpec] = (),
ignore_project_artifact_remotes: bool = False,
):
elements = self._load(
(target,),
selection=_PipelineSelection.NONE,
load_artifacts=True,
attempt_artifact_metadata=True,
connect_artifact_cache=True,
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)

assert len(elements) == 1
element: Element = elements[0]

self._check_location_writable(location, force=force, tar=tar)

self.query_cache([element])
self._pull_missing_artifacts([element])

if buildroot:
# If the user requested the buildroot but no buildroot is available, fail.
if not element._buildroot_exists():
raise StreamError("No buildroot artifact available", reason="missing-buildroot")
else:
# If the user requested the buildtree but no buildtree is available,
# suggest checking out the buildroot if one is available.
# Otherwise, fail if neither is available.
if not element._buildtree_exists():
if element._buildroot_exists():
raise StreamError(
"No buildtree artifact available, but a buildroot is available. "
"Use the --buildroot option to check out the buildroot.",
reason="missing-buildtree-buildroot-exists",
)
raise StreamError(
"Neither a buildtree nor a buildroot artifact is available.",
reason="missing-buildtree-and-buildroot",
)

try:
artifact = element._get_artifact()
virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another thing we need to be aware of (and also give a suggestion about to the user) is that not all elements have a build tree: only build elements do.

So it's not enough to check for the build root, we also need to check for the build tree. Then depending on the availability and the user request, we can suggest an alternative to the user.

buildtree availablebuildroot availableuser requestedresult
YYbuild treeextract build tree
YYbuild rootextract build root
NYbuild treeerror + suggest extract build root
NYbuild rootextract build root
NNbuild treeerror
NNbuild rooterror

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for pointing this out. I've accounted for all of the cases you described and added tests covering them.

Could you please take another look when you have a chance?

self._export_artifact(tar, location, compression, element, hardlinks, virdir)
except BstError as e:
raise StreamError(
"Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason
) from e

# source_checkout()
#
# Checkout sources of the target element to the specified location
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions src/buildstream/_frontend/cli.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -1694,3 +1694,124 @@ def artifact_delete(app, artifacts, deps):
"""Remove artifacts from the local cache"""
with app.initialized():
app.stream.artifact_delete(artifacts, selection=deps)


#############################################################
# Buildtree Commands #
#############################################################
@cli.group(short_help="Manipulate cached buildtree.")
def buildtree():
"""Manipulate cached buildtree"""


#####################################################################
# Buildtree Checkout Command #
#####################################################################
@buildtree.command(name="checkout", short_help="Checkout contents of a buildtree")
@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.")
@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible")
@click.option(
"--tar",
default=None,
metavar="LOCATION",
type=click.Path(),
help="Create a tarball from the artifact contents instead "
"of a file tree. If LOCATION is '-', the tarball "
"will be dumped to the standard output.",
)
@click.option(
"--compression",
default=None,
type=click.Choice(["gz", "xz", "bz2"]),
help="The compression option of the tarball created.",
)
@click.option(
"--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to"
)
@click.option(
"--artifact-remote",
"artifact_remotes",
type=RemoteSpecType(RemoteSpecPurpose.PULL),
multiple=True,
help="A remote for downloading artifacts",
)
@click.option(
"--ignore-project-artifact-remotes",
is_flag=True,
help="Ignore remote artifact cache servers recommended by projects",
)
@click.argument("target", required=False, type=click.Path(readable=False))
@click.pass_obj
def buildtree_checkout(
app,
buildroot,
force,
hardlinks,
tar,
compression,
directory,
artifact_remotes,
ignore_project_artifact_remotes,
target,
):
"""Checkout buildtree

When this command is executed from a workspace directory, the default
is to checkout the artifact of the workspace element.
"""
from .. import utils

if hardlinks and tar:
click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
sys.exit(-1)

if tar and directory:
click.echo("ERROR: options --directory and --tar conflict", err=True)
sys.exit(-1)

if not tar:
if compression:
click.echo("ERROR: --compression can only be provided if --tar is provided", err=True)
sys.exit(-1)
else:
location = tar
try:
inferred_compression = utils._get_compression(tar)
except UtilError as e:
click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True)
sys.exit(-1)
if compression and inferred_compression != "" and inferred_compression != compression:
click.echo(
"WARNING: File extension and compression differ."
"File extension has been overridden by --compression",
err=True,
)
if not compression:
compression = inferred_compression

with app.initialized():
if not target:
target = app.stream.get_default_target()
if not target:
raise AppError('Missing argument "ELEMENT".')

if not tar:
if directory is None:
location = os.path.abspath(os.path.join(os.getcwd(), target))
if location[-4:] == ".bst":
location = location[:-4]
else:
location = directory

app.stream.buildtree_checkout(
target,
location=location,
buildroot=buildroot,
force=force,
hardlinks=hardlinks,
compression=compression,
tar=bool(tar),
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)
78 changes: 78 additions & 0 deletions src/buildstream/_stream.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -916,6 +916,84 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE):
if not ref_removed:
self._context.messenger.info("No artifacts were removed")

# buildtree_checkout()
#
# Checkout target buildtree artifact to the specified location
#
# Args:
# target: Target to checkout
# location: Location to checkout the artifact to
# force: Whether files can be overwritten if necessary
# hardlinks: Whether checking out files hardlinked to
# their artifacts is acceptable
# tar: If true, a tarball from the artifact contents will
# be created, otherwise the file tree of the artifact
# will be placed at the given location. If true and
# location is '-', the tarball will be dumped on the
# standard output.
# artifact_remotes: Artifact cache remotes specified on the commmand line
# ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects
#
def buildtree_checkout(
self,
target: str,
*,
location: Optional[str] = None,
buildroot: bool = False,
force: bool = False,
hardlinks: bool = False,
compression: str = "",
tar: bool = False,
artifact_remotes: Iterable[RemoteSpec] = (),
ignore_project_artifact_remotes: bool = False,
):
elements = self._load(
(target,),
selection=_PipelineSelection.NONE,
load_artifacts=True,
attempt_artifact_metadata=True,
connect_artifact_cache=True,
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)

assert len(elements) == 1
element: Element = elements[0]

self._check_location_writable(location, force=force, tar=tar)

self.query_cache([element])
self._pull_missing_artifacts([element])

if buildroot:
# If the user requested the buildroot but no buildroot is available, fail.
if not element._buildroot_exists():
raise StreamError("No buildroot artifact available", reason="missing-buildroot")
else:
# If the user requested the buildtree but no buildtree is available,
# suggest checking out the buildroot if one is available.
# Otherwise, fail if neither is available.
if not element._buildtree_exists():
if element._buildroot_exists():
raise StreamError(
"No buildtree artifact available, but a buildroot is available. "
"Use the --buildroot option to check out the buildroot.",
reason="missing-buildtree-buildroot-exists",
)
raise StreamError(
"Neither a buildtree nor a buildroot artifact is available.",
reason="missing-buildtree-and-buildroot",
)

try:
artifact = element._get_artifact()
virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another thing we need to be aware of (and also give a suggestion about to the user) is that not all elements have a build tree: only build elements do.

So it's not enough to check for the build root, we also need to check for the build tree. Then depending on the availability and the user request, we can suggest an alternative to the user.

buildtree availablebuildroot availableuser requestedresult
YYbuild treeextract build tree
YYbuild rootextract build root
NYbuild treeerror + suggest extract build root
NYbuild rootextract build root
NNbuild treeerror
NNbuild rooterror

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for pointing this out. I've accounted for all of the cases you described and added tests covering them.

Could you please take another look when you have a chance?

self._export_artifact(tar, location, compression, element, hardlinks, virdir)
except BstError as e:
raise StreamError(
"Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason
) from e

# source_checkout()
#
# Checkout sources of the target element to the specified location
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions src/buildstream/_frontend/cli.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -1694,3 +1694,124 @@ def artifact_delete(app, artifacts, deps):
"""Remove artifacts from the local cache"""
with app.initialized():
app.stream.artifact_delete(artifacts, selection=deps)


#############################################################
# Buildtree Commands #
#############################################################
@cli.group(short_help="Manipulate cached buildtree.")
def buildtree():
"""Manipulate cached buildtree"""


#####################################################################
# Buildtree Checkout Command #
#####################################################################
@buildtree.command(name="checkout", short_help="Checkout contents of a buildtree")
@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.")
@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible")
@click.option(
"--tar",
default=None,
metavar="LOCATION",
type=click.Path(),
help="Create a tarball from the artifact contents instead "
"of a file tree. If LOCATION is '-', the tarball "
"will be dumped to the standard output.",
)
@click.option(
"--compression",
default=None,
type=click.Choice(["gz", "xz", "bz2"]),
help="The compression option of the tarball created.",
)
@click.option(
"--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to"
)
@click.option(
"--artifact-remote",
"artifact_remotes",
type=RemoteSpecType(RemoteSpecPurpose.PULL),
multiple=True,
help="A remote for downloading artifacts",
)
@click.option(
"--ignore-project-artifact-remotes",
is_flag=True,
help="Ignore remote artifact cache servers recommended by projects",
)
@click.argument("target", required=False, type=click.Path(readable=False))
@click.pass_obj
def buildtree_checkout(
app,
buildroot,
force,
hardlinks,
tar,
compression,
directory,
artifact_remotes,
ignore_project_artifact_remotes,
target,
):
"""Checkout buildtree

When this command is executed from a workspace directory, the default
is to checkout the artifact of the workspace element.
"""
from .. import utils

if hardlinks and tar:
click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
sys.exit(-1)

if tar and directory:
click.echo("ERROR: options --directory and --tar conflict", err=True)
sys.exit(-1)

if not tar:
if compression:
click.echo("ERROR: --compression can only be provided if --tar is provided", err=True)
sys.exit(-1)
else:
location = tar
try:
inferred_compression = utils._get_compression(tar)
except UtilError as e:
click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True)
sys.exit(-1)
if compression and inferred_compression != "" and inferred_compression != compression:
click.echo(
"WARNING: File extension and compression differ."
"File extension has been overridden by --compression",
err=True,
)
if not compression:
compression = inferred_compression

with app.initialized():
if not target:
target = app.stream.get_default_target()
if not target:
raise AppError('Missing argument "ELEMENT".')

if not tar:
if directory is None:
location = os.path.abspath(os.path.join(os.getcwd(), target))
if location[-4:] == ".bst":
location = location[:-4]
else:
location = directory

app.stream.buildtree_checkout(
target,
location=location,
buildroot=buildroot,
force=force,
hardlinks=hardlinks,
compression=compression,
tar=bool(tar),
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)
78 changes: 78 additions & 0 deletions src/buildstream/_stream.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -916,6 +916,84 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE):
if not ref_removed:
self._context.messenger.info("No artifacts were removed")

# buildtree_checkout()
#
# Checkout target buildtree artifact to the specified location
#
# Args:
# target: Target to checkout
# location: Location to checkout the artifact to
# force: Whether files can be overwritten if necessary
# hardlinks: Whether checking out files hardlinked to
# their artifacts is acceptable
# tar: If true, a tarball from the artifact contents will
# be created, otherwise the file tree of the artifact
# will be placed at the given location. If true and
# location is '-', the tarball will be dumped on the
# standard output.
# artifact_remotes: Artifact cache remotes specified on the commmand line
# ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects
#
def buildtree_checkout(
self,
target: str,
*,
location: Optional[str] = None,
buildroot: bool = False,
force: bool = False,
hardlinks: bool = False,
compression: str = "",
tar: bool = False,
artifact_remotes: Iterable[RemoteSpec] = (),
ignore_project_artifact_remotes: bool = False,
):
elements = self._load(
(target,),
selection=_PipelineSelection.NONE,
load_artifacts=True,
attempt_artifact_metadata=True,
connect_artifact_cache=True,
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)

assert len(elements) == 1
element: Element = elements[0]

self._check_location_writable(location, force=force, tar=tar)

self.query_cache([element])
self._pull_missing_artifacts([element])

if buildroot:
# If the user requested the buildroot but no buildroot is available, fail.
if not element._buildroot_exists():
raise StreamError("No buildroot artifact available", reason="missing-buildroot")
else:
# If the user requested the buildtree but no buildtree is available,
# suggest checking out the buildroot if one is available.
# Otherwise, fail if neither is available.
if not element._buildtree_exists():
if element._buildroot_exists():
raise StreamError(
"No buildtree artifact available, but a buildroot is available. "
"Use the --buildroot option to check out the buildroot.",
reason="missing-buildtree-buildroot-exists",
)
raise StreamError(
"Neither a buildtree nor a buildroot artifact is available.",
reason="missing-buildtree-and-buildroot",
)

try:
artifact = element._get_artifact()
virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another thing we need to be aware of (and also give a suggestion about to the user) is that not all elements have a build tree: only build elements do.

So it's not enough to check for the build root, we also need to check for the build tree. Then depending on the availability and the user request, we can suggest an alternative to the user.

buildtree availablebuildroot availableuser requestedresult
YYbuild treeextract build tree
YYbuild rootextract build root
NYbuild treeerror + suggest extract build root
NYbuild rootextract build root
NNbuild treeerror
NNbuild rooterror

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for pointing this out. I've accounted for all of the cases you described and added tests covering them.

Could you please take another look when you have a chance?

self._export_artifact(tar, location, compression, element, hardlinks, virdir)
except BstError as e:
raise StreamError(
"Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason
) from e

# source_checkout()
#
# Checkout sources of the target element to the specified location
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions src/buildstream/_frontend/cli.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -1694,3 +1694,124 @@ def artifact_delete(app, artifacts, deps):
"""Remove artifacts from the local cache"""
with app.initialized():
app.stream.artifact_delete(artifacts, selection=deps)


#############################################################
# Buildtree Commands #
#############################################################
@cli.group(short_help="Manipulate cached buildtree.")
def buildtree():
"""Manipulate cached buildtree"""


#####################################################################
# Buildtree Checkout Command #
#####################################################################
@buildtree.command(name="checkout", short_help="Checkout contents of a buildtree")
@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.")
@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible")
@click.option(
"--tar",
default=None,
metavar="LOCATION",
type=click.Path(),
help="Create a tarball from the artifact contents instead "
"of a file tree. If LOCATION is '-', the tarball "
"will be dumped to the standard output.",
)
@click.option(
"--compression",
default=None,
type=click.Choice(["gz", "xz", "bz2"]),
help="The compression option of the tarball created.",
)
@click.option(
"--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to"
)
@click.option(
"--artifact-remote",
"artifact_remotes",
type=RemoteSpecType(RemoteSpecPurpose.PULL),
multiple=True,
help="A remote for downloading artifacts",
)
@click.option(
"--ignore-project-artifact-remotes",
is_flag=True,
help="Ignore remote artifact cache servers recommended by projects",
)
@click.argument("target", required=False, type=click.Path(readable=False))
@click.pass_obj
def buildtree_checkout(
app,
buildroot,
force,
hardlinks,
tar,
compression,
directory,
artifact_remotes,
ignore_project_artifact_remotes,
target,
):
"""Checkout buildtree

When this command is executed from a workspace directory, the default
is to checkout the artifact of the workspace element.
"""
from .. import utils

if hardlinks and tar:
click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
sys.exit(-1)

if tar and directory:
click.echo("ERROR: options --directory and --tar conflict", err=True)
sys.exit(-1)

if not tar:
if compression:
click.echo("ERROR: --compression can only be provided if --tar is provided", err=True)
sys.exit(-1)
else:
location = tar
try:
inferred_compression = utils._get_compression(tar)
except UtilError as e:
click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True)
sys.exit(-1)
if compression and inferred_compression != "" and inferred_compression != compression:
click.echo(
"WARNING: File extension and compression differ."
"File extension has been overridden by --compression",
err=True,
)
if not compression:
compression = inferred_compression

with app.initialized():
if not target:
target = app.stream.get_default_target()
if not target:
raise AppError('Missing argument "ELEMENT".')

if not tar:
if directory is None:
location = os.path.abspath(os.path.join(os.getcwd(), target))
if location[-4:] == ".bst":
location = location[:-4]
else:
location = directory

app.stream.buildtree_checkout(
target,
location=location,
buildroot=buildroot,
force=force,
hardlinks=hardlinks,
compression=compression,
tar=bool(tar),
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)
78 changes: 78 additions & 0 deletions src/buildstream/_stream.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -916,6 +916,84 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE):
if not ref_removed:
self._context.messenger.info("No artifacts were removed")

# buildtree_checkout()
#
# Checkout target buildtree artifact to the specified location
#
# Args:
# target: Target to checkout
# location: Location to checkout the artifact to
# force: Whether files can be overwritten if necessary
# hardlinks: Whether checking out files hardlinked to
# their artifacts is acceptable
# tar: If true, a tarball from the artifact contents will
# be created, otherwise the file tree of the artifact
# will be placed at the given location. If true and
# location is '-', the tarball will be dumped on the
# standard output.
# artifact_remotes: Artifact cache remotes specified on the commmand line
# ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects
#
def buildtree_checkout(
self,
target: str,
*,
location: Optional[str] = None,
buildroot: bool = False,
force: bool = False,
hardlinks: bool = False,
compression: str = "",
tar: bool = False,
artifact_remotes: Iterable[RemoteSpec] = (),
ignore_project_artifact_remotes: bool = False,
):
elements = self._load(
(target,),
selection=_PipelineSelection.NONE,
load_artifacts=True,
attempt_artifact_metadata=True,
connect_artifact_cache=True,
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)

assert len(elements) == 1
element: Element = elements[0]

self._check_location_writable(location, force=force, tar=tar)

self.query_cache([element])
self._pull_missing_artifacts([element])

if buildroot:
# If the user requested the buildroot but no buildroot is available, fail.
if not element._buildroot_exists():
raise StreamError("No buildroot artifact available", reason="missing-buildroot")
else:
# If the user requested the buildtree but no buildtree is available,
# suggest checking out the buildroot if one is available.
# Otherwise, fail if neither is available.
if not element._buildtree_exists():
if element._buildroot_exists():
raise StreamError(
"No buildtree artifact available, but a buildroot is available. "
"Use the --buildroot option to check out the buildroot.",
reason="missing-buildtree-buildroot-exists",
)
raise StreamError(
"Neither a buildtree nor a buildroot artifact is available.",
reason="missing-buildtree-and-buildroot",
)

try:
artifact = element._get_artifact()
virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another thing we need to be aware of (and also give a suggestion about to the user) is that not all elements have a build tree: only build elements do.

So it's not enough to check for the build root, we also need to check for the build tree. Then depending on the availability and the user request, we can suggest an alternative to the user.

buildtree availablebuildroot availableuser requestedresult
YYbuild treeextract build tree
YYbuild rootextract build root
NYbuild treeerror + suggest extract build root
NYbuild rootextract build root
NNbuild treeerror
NNbuild rooterror

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for pointing this out. I've accounted for all of the cases you described and added tests covering them.

Could you please take another look when you have a chance?

self._export_artifact(tar, location, compression, element, hardlinks, virdir)
except BstError as e:
raise StreamError(
"Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason
) from e

# source_checkout()
#
# Checkout sources of the target element to the specified location
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions src/buildstream/_frontend/cli.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -1694,3 +1694,124 @@ def artifact_delete(app, artifacts, deps):
"""Remove artifacts from the local cache"""
with app.initialized():
app.stream.artifact_delete(artifacts, selection=deps)


#############################################################
# Buildtree Commands #
#############################################################
@cli.group(short_help="Manipulate cached buildtree.")
def buildtree():
"""Manipulate cached buildtree"""


#####################################################################
# Buildtree Checkout Command #
#####################################################################
@buildtree.command(name="checkout", short_help="Checkout contents of a buildtree")
@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.")
@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible")
@click.option(
"--tar",
default=None,
metavar="LOCATION",
type=click.Path(),
help="Create a tarball from the artifact contents instead "
"of a file tree. If LOCATION is '-', the tarball "
"will be dumped to the standard output.",
)
@click.option(
"--compression",
default=None,
type=click.Choice(["gz", "xz", "bz2"]),
help="The compression option of the tarball created.",
)
@click.option(
"--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to"
)
@click.option(
"--artifact-remote",
"artifact_remotes",
type=RemoteSpecType(RemoteSpecPurpose.PULL),
multiple=True,
help="A remote for downloading artifacts",
)
@click.option(
"--ignore-project-artifact-remotes",
is_flag=True,
help="Ignore remote artifact cache servers recommended by projects",
)
@click.argument("target", required=False, type=click.Path(readable=False))
@click.pass_obj
def buildtree_checkout(
app,
buildroot,
force,
hardlinks,
tar,
compression,
directory,
artifact_remotes,
ignore_project_artifact_remotes,
target,
):
"""Checkout buildtree

When this command is executed from a workspace directory, the default
is to checkout the artifact of the workspace element.
"""
from .. import utils

if hardlinks and tar:
click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
sys.exit(-1)

if tar and directory:
click.echo("ERROR: options --directory and --tar conflict", err=True)
sys.exit(-1)

if not tar:
if compression:
click.echo("ERROR: --compression can only be provided if --tar is provided", err=True)
sys.exit(-1)
else:
location = tar
try:
inferred_compression = utils._get_compression(tar)
except UtilError as e:
click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True)
sys.exit(-1)
if compression and inferred_compression != "" and inferred_compression != compression:
click.echo(
"WARNING: File extension and compression differ."
"File extension has been overridden by --compression",
err=True,
)
if not compression:
compression = inferred_compression

with app.initialized():
if not target:
target = app.stream.get_default_target()
if not target:
raise AppError('Missing argument "ELEMENT".')

if not tar:
if directory is None:
location = os.path.abspath(os.path.join(os.getcwd(), target))
if location[-4:] == ".bst":
location = location[:-4]
else:
location = directory

app.stream.buildtree_checkout(
target,
location=location,
buildroot=buildroot,
force=force,
hardlinks=hardlinks,
compression=compression,
tar=bool(tar),
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)
78 changes: 78 additions & 0 deletions src/buildstream/_stream.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -916,6 +916,84 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE):
if not ref_removed:
self._context.messenger.info("No artifacts were removed")

# buildtree_checkout()
#
# Checkout target buildtree artifact to the specified location
#
# Args:
# target: Target to checkout
# location: Location to checkout the artifact to
# force: Whether files can be overwritten if necessary
# hardlinks: Whether checking out files hardlinked to
# their artifacts is acceptable
# tar: If true, a tarball from the artifact contents will
# be created, otherwise the file tree of the artifact
# will be placed at the given location. If true and
# location is '-', the tarball will be dumped on the
# standard output.
# artifact_remotes: Artifact cache remotes specified on the commmand line
# ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects
#
def buildtree_checkout(
self,
target: str,
*,
location: Optional[str] = None,
buildroot: bool = False,
force: bool = False,
hardlinks: bool = False,
compression: str = "",
tar: bool = False,
artifact_remotes: Iterable[RemoteSpec] = (),
ignore_project_artifact_remotes: bool = False,
):
elements = self._load(
(target,),
selection=_PipelineSelection.NONE,
load_artifacts=True,
attempt_artifact_metadata=True,
connect_artifact_cache=True,
artifact_remotes=artifact_remotes,
ignore_project_artifact_remotes=ignore_project_artifact_remotes,
)

assert len(elements) == 1
element: Element = elements[0]

self._check_location_writable(location, force=force, tar=tar)

self.query_cache([element])
self._pull_missing_artifacts([element])

if buildroot:
# If the user requested the buildroot but no buildroot is available, fail.
if not element._buildroot_exists():
raise StreamError("No buildroot artifact available", reason="missing-buildroot")
else:
# If the user requested the buildtree but no buildtree is available,
# suggest checking out the buildroot if one is available.
# Otherwise, fail if neither is available.
if not element._buildtree_exists():
if element._buildroot_exists():
raise StreamError(
"No buildtree artifact available, but a buildroot is available. "
"Use the --buildroot option to check out the buildroot.",
reason="missing-buildtree-buildroot-exists",
)
raise StreamError(
"Neither a buildtree nor a buildroot artifact is available.",
reason="missing-buildtree-and-buildroot",
)

try:
artifact = element._get_artifact()
virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another thing we need to be aware of (and also give a suggestion about to the user) is that not all elements have a build tree: only build elements do.

So it's not enough to check for the build root, we also need to check for the build tree. Then depending on the availability and the user request, we can suggest an alternative to the user.

buildtree availablebuildroot availableuser requestedresult
YYbuild treeextract build tree
YYbuild rootextract build root
NYbuild treeerror + suggest extract build root
NYbuild rootextract build root
NNbuild treeerror
NNbuild rooterror

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for pointing this out. I've accounted for all of the cases you described and added tests covering them.

Could you please take another look when you have a chance?

self._export_artifact(tar, location, compression, element, hardlinks, virdir)
except BstError as e:
raise StreamError(
"Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason
) from e

# source_checkout()
#
# Checkout sources of the target element to the specified location
Expand Down
Loading