This repository was archived by the owner on Aug 12, 2026. It is now read-only.

Removed dependency on docker#21

Draft
Bulkin wants to merge 3 commits into
base:main
Choose a base branch
from
Draft

Removed dependency on docker #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a4bdab4
Removed dependency on docker, added `toltecmk-contained` script...
Bulkin Aug 12, 2021
0ec133d
Merge branch 'main' into no-docker
matteodelabre Sep 1, 2021
4ca6575
Merge branch 'main' into no-docker
matteodelabre Sep 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion toltec/__main__.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,16 @@ def main() -> int:
(default: [current directory]/dist)""",
)

parser.add_argument(
"-s",
"--source-dir",
metavar="DIR",
default=None,
help="""path to the source directory
(optional: when specified, a local build is performed instead of fetching
sources)""",
)

parser.add_argument(
"-a",
"--arch-name",
Expand DownExpand Up@@ -80,7 +90,7 @@ def main() -> int:

recipe_bundle = parse_recipe(args.recipe_dir)

with Builder(args.work_dir, args.dist_dir) as builder:
with Builder(args.work_dir, args.dist_dir, args.source_dir) as builder:
if args.hook:
for ident in args.hook:
if ident and ident[0] in (".", "/"):
Expand Down
54 changes: 0 additions & 54 deletions toltec/bash.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,7 +8,6 @@
import logging
from collections import deque
from typing import Deque, Dict, Generator, List, Optional, Tuple, Union
from docker.client import DockerClient

AssociativeArray = Dict[str, str]
IndexedArray = List[Optional[str]]
Expand DownExpand Up@@ -363,59 +362,6 @@ def run_script(variables: Variables, script: str) -> LogGenerator:
raise ScriptError(f"Script exited with code {process.returncode}")


def run_script_in_container(
docker: DockerClient,
image: str,
mounts: List,
variables: Variables,
script: str,
) -> LogGenerator:
"""
Run a Bash script inside a Docker container and stream its output.

:param docker: Docker client
:param image: image to use for the new container
:param mounts: paths to mount in the container
:param variables: Bash variables to set before running the script
:param script: Bash script to execute
:returns: generator yielding output lines from the script
:raises ScriptError: if the script exits with a non-zero code
"""
container = docker.containers.run(
image,
mounts=mounts,
command=[
"/usr/bin/env",
"bash",
"-c",
"\n".join(
(
"set -euo pipefail",
put_variables(variables),
"script() {",
script,
"}",
"script",
)
),
],
detach=True,
security_opt=["label=disable"],
)

try:
for line in container.logs(stream=True):
if line:
yield line.decode().strip()

result = container.wait()

if result["StatusCode"] != 0:
raise ScriptError(f"Script exited with code {result['StatusCode']}")
finally:
container.remove()


def pipe_logs(
logger: logging.Logger,
logs: LogGenerator,
Expand Down
62 changes: 17 additions & 45 deletions toltec/builder.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -9,7 +9,6 @@
import os
import logging
import textwrap
import docker
import requests
from . import bash, util, ipk
from .recipe import RecipeBundle, Recipe, Package
Expand All@@ -28,10 +27,9 @@ class Builder: # pylint: disable=too-few-public-methods
# Detect non-local paths
URL_REGEX = re.compile(r"[a-z]+://")

# Prefix for all Toltec Docker images
IMAGE_PREFIX = "ghcr.io/toltec-dev/"

def __init__(self, work_dir: str, dist_dir: str) -> None:
def __init__(
self, work_dir: str, dist_dir: str, source_dir: Optional[str]
) -> None:
"""
Create a builder helper.

Expand All@@ -40,15 +38,7 @@ def __init__(self, work_dir: str, dist_dir: str) -> None:
"""
self.work_dir = work_dir
self.dist_dir = dist_dir

try:
self.docker = docker.from_env()
except docker.errors.DockerException as err:
raise BuildError(
"Unable to connect to the Docker daemon. \
Please check that the service is running and that you have the necessary \
permissions."
) from err
self.source_dir = source_dir

def __enter__(self) -> "Builder":
return self
Expand All@@ -59,7 +49,7 @@ def __exit__(
exc_value: Optional[BaseException],
traceback: Optional[TracebackType],
) -> None:
self.docker.close()
pass

@util.hook
def post_parse(self, recipe: Recipe) -> None:
Expand DownExpand Up@@ -164,13 +154,13 @@ def _make_arch(
packages: Optional[List[Package]] = None,
) -> bool:
self.post_parse(recipe)

src_dir = os.path.join(build_dir, "src")
os.makedirs(src_dir, exist_ok=True)

self._fetch_sources(recipe, src_dir)
self.post_fetch_sources(recipe, src_dir)

if self.source_dir:
src_dir = self.source_dir
else:
src_dir = os.path.join(build_dir, "src")
os.makedirs(src_dir, exist_ok=True)
self._fetch_sources(recipe, src_dir)
self.post_fetch_sources(recipe, src_dir)
self._prepare(recipe, src_dir)
self.post_prepare(recipe, src_dir)

Expand DownExpand Up@@ -274,8 +264,6 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
for filename in util.list_tree(src_dir):
os.utime(filename, (epoch, epoch))

mount_src = "/src"
repo_src = "/repo"
uid = os.getuid()
pre_script: List[str] = []

Expand DownExpand Up@@ -331,33 +319,17 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
" -- " + " ".join(host_deps),
)
)

logs = bash.run_script_in_container(
self.docker,
image=self.IMAGE_PREFIX + recipe.image,
mounts=[
docker.types.Mount(
type="bind",
source=os.path.abspath(src_dir),
target=mount_src,
),
docker.types.Mount(
type="bind",
source=os.path.abspath(self.dist_dir),
target=repo_src,
),
],
variables={
"srcdir": mount_src,
},
logs = bash.run_script(
script="\n".join(
(
f'cd "{src_dir}"',
*pre_script,
f'cd "{mount_src}"',
recipe.build,
f'chown -R {uid}:{uid} "{mount_src}"',
)
),
variables={
"srcdir": src_dir,
},
)
bash.pipe_logs(logger, logs, "build()")

Expand Down
23 changes: 11 additions & 12 deletions toltec/util.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,7 +19,7 @@
IO,
List,
Optional,
Protocol,
# Protocol,
Type,
Union,
)
Expand DownExpand Up@@ -302,6 +302,8 @@ def check_directory(path: str, message: str) -> bool:
try:
os.mkdir(path)
except FileExistsError:
if not os.listdir(path):
return True
ans = query_user(
message,
default="c",
Expand DownExpand Up@@ -343,18 +345,15 @@ def list_tree(root: str) -> List[str]:
HookTrigger = Callable[..., None]
HookListener = Callable[..., None]

# Protocol is not available in python 3.7 (Debian buster)
Hook = Any
# class Hook(Protocol): # pylint:disable=too-few-public-methods
# """Protocol for hooks."""

class Hook(Protocol): # pylint:disable=too-few-public-methods
"""Protocol for hooks."""

@staticmethod
def register(new_listener: HookListener) -> None:
"""Add a new listener to this hook."""
...

# Invoke all listeners for this hook
__call__: HookTrigger

# @staticmethod
# def register(new_listener: HookListener) -> None:
# """Add a new listener to this hook."""
# ...

def hook(func: HookTrigger) -> Hook:
"""
Expand Down
1 change: 1 addition & 0 deletions toltecmk
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
toltecmk.py
59 changes: 59 additions & 0 deletions toltecmk-contained
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
# Copyright (c) 2021 The Toltec Contributors
# SPDX-License-Identifier: MIT

import argparse
import os
import subprocess
from pathlib import Path

from toltec import parse_recipe
from toltecmk import make_argparser

# Prefix for all Toltec Docker images
IMAGE_PREFIX = "ghcr.io/toltec-dev/"

args = make_argparser().parse_args()

recipe_bundle = parse_recipe(args.recipe_dir)
# Don't recipes in a bundle share most information?
recipe = next(iter(recipe_bundle.values()))

image = IMAGE_PREFIX + recipe.image

toltecmk_dir = str(Path(__file__).resolve().parent)

mounts = [ (args.recipe_dir, '/recipe'),
(args.work_dir, '/build'),
(args.dist_dir, '/pkg'),
(toltecmk_dir, '/toltecmk'),
]

for m in mounts:
if not os.path.exists(m[0]):
os.makedirs(m[0])

if args.source_dir:
mounts.append((args.source_dir, '/src'))

cmd = ['podman', 'run', '-it', '--rm']
for m in mounts:
cmd.append('-v')
cmd.append(':'.join(m))
cmd.append(image)
cmd += ['bash', '-c']
# currently required by toltecmk and missing in images
internal_cmd = []
internal_cmd += ['apt update && apt install -yq python3-dateutil python3-requests && ']
internal_cmd += ['/toltecmk/toltecmk',
'-w', '/build',
'-d', '/pkg',
'/recipe']
if args.source_dir: internal_cmd += ['-s', '/src']
if args.arch_name: internal_cmd += ['-a', args.arch_name]
if args.package_name: internal_cmd += ['-p', args.package_name]

cmd.append(' '.join(internal_cmd))
print(' '.join('"' + c + '"' if ' ' in c else c for c in cmd))
subprocess.call(cmd)

Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content

This repository was archived by the owner on Aug 12, 2026. It is now read-only.

Removed dependency on docker#21
 Draft

Bulkin
wants to merge
3
commits into

base:main
Choose a base branch

from

 Draft


Removed dependency on docker
#21
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
a4bdab4

Removed dependency on docker, added `toltecmk-contained` script...


Bulkin Aug 12, 2021
0ec133d

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021
4ca6575

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 11 additions & 1 deletion
toltec/__main__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,16 @@ def main() -> int:
 (default: [current directory]/dist)""",
 )

 parser.add_argument(
 "-s",
 "--source-dir",
 metavar="DIR",
 default=None,
 help="""path to the source directory
 (optional: when specified, a local build is performed instead of fetching 
 sources)""",
 )

 parser.add_argument(
 "-a",
 "--arch-name",
Expand DownExpand Up@@ -80,7 +90,7 @@ def main() -> int:

 recipe_bundle = parse_recipe(args.recipe_dir)

 with Builder(args.work_dir, args.dist_dir) as builder:
 with Builder(args.work_dir, args.dist_dir, args.source_dir) as builder:
 if args.hook:
 for ident in args.hook:
 if ident and ident[0] in (".", "/"):
Expand Down

54 changes: 0 additions & 54 deletions
toltec/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,7 +8,6 @@
import logging
from collections import deque
from typing import Deque, Dict, Generator, List, Optional, Tuple, Union
from docker.client import DockerClient

AssociativeArray = Dict[str, str]
IndexedArray = List[Optional[str]]
Expand DownExpand Up@@ -363,59 +362,6 @@ def run_script(variables: Variables, script: str) -> LogGenerator:
 raise ScriptError(f"Script exited with code {process.returncode}")


def run_script_in_container(
 docker: DockerClient,
 image: str,
 mounts: List,
 variables: Variables,
 script: str,
) -> LogGenerator:
 """
 Run a Bash script inside a Docker container and stream its output.

 :param docker: Docker client
 :param image: image to use for the new container
 :param mounts: paths to mount in the container
 :param variables: Bash variables to set before running the script
 :param script: Bash script to execute
 :returns: generator yielding output lines from the script
 :raises ScriptError: if the script exits with a non-zero code
 """
 container = docker.containers.run(
 image,
 mounts=mounts,
 command=[
 "/usr/bin/env",
 "bash",
 "-c",
 "\n".join(
 (
 "set -euo pipefail",
 put_variables(variables),
 "script() {",
 script,
 "}",
 "script",
 )
 ),
 ],
 detach=True,
 security_opt=["label=disable"],
 )

 try:
 for line in container.logs(stream=True):
 if line:
 yield line.decode().strip()

 result = container.wait()

 if result["StatusCode"] != 0:
 raise ScriptError(f"Script exited with code {result['StatusCode']}")
 finally:
 container.remove()


def pipe_logs(
 logger: logging.Logger,
 logs: LogGenerator,
Expand Down

62 changes: 17 additions & 45 deletions
toltec/builder.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,7 +9,6 @@
import os
import logging
import textwrap
import docker
import requests
from . import bash, util, ipk
from .recipe import RecipeBundle, Recipe, Package
Expand All@@ -28,10 +27,9 @@ class Builder: # pylint: disable=too-few-public-methods
 # Detect non-local paths
 URL_REGEX = re.compile(r"[a-z]+://")

 # Prefix for all Toltec Docker images
 IMAGE_PREFIX = "ghcr.io/toltec-dev/"

 def __init__(self, work_dir: str, dist_dir: str) -> None:
 def __init__(
 self, work_dir: str, dist_dir: str, source_dir: Optional[str]
 ) -> None:
 """
 Create a builder helper.

Expand All@@ -40,15 +38,7 @@ def __init__(self, work_dir: str, dist_dir: str) -> None:
 """
 self.work_dir = work_dir
 self.dist_dir = dist_dir

 try:
 self.docker = docker.from_env()
 except docker.errors.DockerException as err:
 raise BuildError(
 "Unable to connect to the Docker daemon. \
Please check that the service is running and that you have the necessary \
permissions."
 ) from err
 self.source_dir = source_dir

 def __enter__(self) -> "Builder":
 return self
Expand All@@ -59,7 +49,7 @@ def __exit__(
 exc_value: Optional[BaseException],
 traceback: Optional[TracebackType],
 ) -> None:
self.docker.close()
pass

 @util.hook
 def post_parse(self, recipe: Recipe) -> None:
Expand DownExpand Up@@ -164,13 +154,13 @@ def _make_arch(
 packages: Optional[List[Package]] = None,
 ) -> bool:
 self.post_parse(recipe)

 src_dir = os.path.join(build_dir, "src")
os.makedirs(src_dir, exist_ok=True)

self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)

 if self.source_dir:
src_dir = self.source_dir
else:
 src_dir = os.path.join(build_dir, "src")
 os.makedirs(src_dir, exist_ok=True)
self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)
 self._prepare(recipe, src_dir)
 self.post_prepare(recipe, src_dir)

Expand DownExpand Up@@ -274,8 +264,6 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 for filename in util.list_tree(src_dir):
 os.utime(filename, (epoch, epoch))

 mount_src = "/src"
 repo_src = "/repo"
 uid = os.getuid()
 pre_script: List[str] = []

Expand DownExpand Up@@ -331,33 +319,17 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 " -- " + " ".join(host_deps),
 )
 )

 logs = bash.run_script_in_container(
 self.docker,
 image=self.IMAGE_PREFIX + recipe.image,
 mounts=[
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(src_dir),
 target=mount_src,
 ),
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(self.dist_dir),
 target=repo_src,
 ),
 ],
 variables={
 "srcdir": mount_src,
 },
 logs = bash.run_script(
 script="\n".join(
 (
 f'cd "{src_dir}"',
 *pre_script,
 f'cd "{mount_src}"',
 recipe.build,
 f'chown -R {uid}:{uid} "{mount_src}"',
 )
 ),
 variables={
 "srcdir": src_dir,
 },
 )
 bash.pipe_logs(logger, logs, "build()")

Expand Down

23 changes: 11 additions & 12 deletions
toltec/util.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,7 +19,7 @@
 IO,
 List,
 Optional,
 Protocol,
# Protocol,
 Type,
 Union,
)
Expand DownExpand Up@@ -302,6 +302,8 @@ def check_directory(path: str, message: str) -> bool:
 try:
 os.mkdir(path)
 except FileExistsError:
 if not os.listdir(path):
 return True
 ans = query_user(
 message,
 default="c",
Expand DownExpand Up@@ -343,18 +345,15 @@ def list_tree(root: str) -> List[str]:
HookTrigger = Callable[..., None]
HookListener = Callable[..., None]

# Protocol is not available in python 3.7 (Debian buster)
Hook = Any
# class Hook(Protocol): # pylint:disable=too-few-public-methods
# """Protocol for hooks."""

class Hook(Protocol): # pylint:disable=too-few-public-methods
 """Protocol for hooks."""

 @staticmethod
 def register(new_listener: HookListener) -> None:
 """Add a new listener to this hook."""
 ...

 # Invoke all listeners for this hook
 __call__: HookTrigger

# @staticmethod
# def register(new_listener: HookListener) -> None:
# """Add a new listener to this hook."""
# ...

def hook(func: HookTrigger) -> Hook:
 """
Expand Down

1 change: 1 addition & 0 deletions
toltecmk

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
toltecmk.py

59 changes: 59 additions & 0 deletions
toltecmk-contained

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
# Copyright (c) 2021 The Toltec Contributors
# SPDX-License-Identifier: MIT

import argparse
import os
import subprocess
from pathlib import Path

from toltec import parse_recipe
from toltecmk import make_argparser

# Prefix for all Toltec Docker images
IMAGE_PREFIX = "ghcr.io/toltec-dev/"

args = make_argparser().parse_args()

recipe_bundle = parse_recipe(args.recipe_dir)
# Don't recipes in a bundle share most information?
recipe = next(iter(recipe_bundle.values()))

image = IMAGE_PREFIX + recipe.image

toltecmk_dir = str(Path(__file__).resolve().parent)

mounts = [ (args.recipe_dir, '/recipe'),
 (args.work_dir, '/build'),
 (args.dist_dir, '/pkg'),
 (toltecmk_dir, '/toltecmk'),
 ]

for m in mounts:
 if not os.path.exists(m[0]):
 os.makedirs(m[0])

if args.source_dir:
 mounts.append((args.source_dir, '/src'))

cmd = ['podman', 'run', '-it', '--rm']
for m in mounts:
 cmd.append('-v')
 cmd.append(':'.join(m))
cmd.append(image)
cmd += ['bash', '-c']
# currently required by toltecmk and missing in images
internal_cmd = []
internal_cmd += ['apt update && apt install -yq python3-dateutil python3-requests && ']
internal_cmd += ['/toltecmk/toltecmk',
 '-w', '/build',
 '-d', '/pkg',
 '/recipe']
if args.source_dir: internal_cmd += ['-s', '/src']
if args.arch_name: internal_cmd += ['-a', args.arch_name]
if args.package_name: internal_cmd += ['-p', args.package_name]

cmd.append(' '.join(internal_cmd))
print(' '.join('"' + c + '"' if ' ' in c else c for c in cmd))
subprocess.call(cmd)

Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content

This repository was archived by the owner on Aug 12, 2026. It is now read-only.

Removed dependency on docker#21
 Draft

Bulkin
wants to merge
3
commits into

base:main
Choose a base branch

from

 Draft


Removed dependency on docker
#21
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
a4bdab4

Removed dependency on docker, added `toltecmk-contained` script...


Bulkin Aug 12, 2021
0ec133d

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021
4ca6575

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 11 additions & 1 deletion
toltec/__main__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,16 @@ def main() -> int:
 (default: [current directory]/dist)""",
 )

 parser.add_argument(
 "-s",
 "--source-dir",
 metavar="DIR",
 default=None,
 help="""path to the source directory
 (optional: when specified, a local build is performed instead of fetching 
 sources)""",
 )

 parser.add_argument(
 "-a",
 "--arch-name",
Expand DownExpand Up@@ -80,7 +90,7 @@ def main() -> int:

 recipe_bundle = parse_recipe(args.recipe_dir)

 with Builder(args.work_dir, args.dist_dir) as builder:
 with Builder(args.work_dir, args.dist_dir, args.source_dir) as builder:
 if args.hook:
 for ident in args.hook:
 if ident and ident[0] in (".", "/"):
Expand Down

54 changes: 0 additions & 54 deletions
toltec/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,7 +8,6 @@
import logging
from collections import deque
from typing import Deque, Dict, Generator, List, Optional, Tuple, Union
from docker.client import DockerClient

AssociativeArray = Dict[str, str]
IndexedArray = List[Optional[str]]
Expand DownExpand Up@@ -363,59 +362,6 @@ def run_script(variables: Variables, script: str) -> LogGenerator:
 raise ScriptError(f"Script exited with code {process.returncode}")


def run_script_in_container(
 docker: DockerClient,
 image: str,
 mounts: List,
 variables: Variables,
 script: str,
) -> LogGenerator:
 """
 Run a Bash script inside a Docker container and stream its output.

 :param docker: Docker client
 :param image: image to use for the new container
 :param mounts: paths to mount in the container
 :param variables: Bash variables to set before running the script
 :param script: Bash script to execute
 :returns: generator yielding output lines from the script
 :raises ScriptError: if the script exits with a non-zero code
 """
 container = docker.containers.run(
 image,
 mounts=mounts,
 command=[
 "/usr/bin/env",
 "bash",
 "-c",
 "\n".join(
 (
 "set -euo pipefail",
 put_variables(variables),
 "script() {",
 script,
 "}",
 "script",
 )
 ),
 ],
 detach=True,
 security_opt=["label=disable"],
 )

 try:
 for line in container.logs(stream=True):
 if line:
 yield line.decode().strip()

 result = container.wait()

 if result["StatusCode"] != 0:
 raise ScriptError(f"Script exited with code {result['StatusCode']}")
 finally:
 container.remove()


def pipe_logs(
 logger: logging.Logger,
 logs: LogGenerator,
Expand Down

62 changes: 17 additions & 45 deletions
toltec/builder.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,7 +9,6 @@
import os
import logging
import textwrap
import docker
import requests
from . import bash, util, ipk
from .recipe import RecipeBundle, Recipe, Package
Expand All@@ -28,10 +27,9 @@ class Builder: # pylint: disable=too-few-public-methods
 # Detect non-local paths
 URL_REGEX = re.compile(r"[a-z]+://")

 # Prefix for all Toltec Docker images
 IMAGE_PREFIX = "ghcr.io/toltec-dev/"

 def __init__(self, work_dir: str, dist_dir: str) -> None:
 def __init__(
 self, work_dir: str, dist_dir: str, source_dir: Optional[str]
 ) -> None:
 """
 Create a builder helper.

Expand All@@ -40,15 +38,7 @@ def __init__(self, work_dir: str, dist_dir: str) -> None:
 """
 self.work_dir = work_dir
 self.dist_dir = dist_dir

 try:
 self.docker = docker.from_env()
 except docker.errors.DockerException as err:
 raise BuildError(
 "Unable to connect to the Docker daemon. \
Please check that the service is running and that you have the necessary \
permissions."
 ) from err
 self.source_dir = source_dir

 def __enter__(self) -> "Builder":
 return self
Expand All@@ -59,7 +49,7 @@ def __exit__(
 exc_value: Optional[BaseException],
 traceback: Optional[TracebackType],
 ) -> None:
self.docker.close()
pass

 @util.hook
 def post_parse(self, recipe: Recipe) -> None:
Expand DownExpand Up@@ -164,13 +154,13 @@ def _make_arch(
 packages: Optional[List[Package]] = None,
 ) -> bool:
 self.post_parse(recipe)

 src_dir = os.path.join(build_dir, "src")
os.makedirs(src_dir, exist_ok=True)

self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)

 if self.source_dir:
src_dir = self.source_dir
else:
 src_dir = os.path.join(build_dir, "src")
 os.makedirs(src_dir, exist_ok=True)
self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)
 self._prepare(recipe, src_dir)
 self.post_prepare(recipe, src_dir)

Expand DownExpand Up@@ -274,8 +264,6 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 for filename in util.list_tree(src_dir):
 os.utime(filename, (epoch, epoch))

 mount_src = "/src"
 repo_src = "/repo"
 uid = os.getuid()
 pre_script: List[str] = []

Expand DownExpand Up@@ -331,33 +319,17 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 " -- " + " ".join(host_deps),
 )
 )

 logs = bash.run_script_in_container(
 self.docker,
 image=self.IMAGE_PREFIX + recipe.image,
 mounts=[
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(src_dir),
 target=mount_src,
 ),
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(self.dist_dir),
 target=repo_src,
 ),
 ],
 variables={
 "srcdir": mount_src,
 },
 logs = bash.run_script(
 script="\n".join(
 (
 f'cd "{src_dir}"',
 *pre_script,
 f'cd "{mount_src}"',
 recipe.build,
 f'chown -R {uid}:{uid} "{mount_src}"',
 )
 ),
 variables={
 "srcdir": src_dir,
 },
 )
 bash.pipe_logs(logger, logs, "build()")

Expand Down

23 changes: 11 additions & 12 deletions
toltec/util.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,7 +19,7 @@
 IO,
 List,
 Optional,
 Protocol,
# Protocol,
 Type,
 Union,
)
Expand DownExpand Up@@ -302,6 +302,8 @@ def check_directory(path: str, message: str) -> bool:
 try:
 os.mkdir(path)
 except FileExistsError:
 if not os.listdir(path):
 return True
 ans = query_user(
 message,
 default="c",
Expand DownExpand Up@@ -343,18 +345,15 @@ def list_tree(root: str) -> List[str]:
HookTrigger = Callable[..., None]
HookListener = Callable[..., None]

# Protocol is not available in python 3.7 (Debian buster)
Hook = Any
# class Hook(Protocol): # pylint:disable=too-few-public-methods
# """Protocol for hooks."""

class Hook(Protocol): # pylint:disable=too-few-public-methods
 """Protocol for hooks."""

 @staticmethod
 def register(new_listener: HookListener) -> None:
 """Add a new listener to this hook."""
 ...

 # Invoke all listeners for this hook
 __call__: HookTrigger

# @staticmethod
# def register(new_listener: HookListener) -> None:
# """Add a new listener to this hook."""
# ...

def hook(func: HookTrigger) -> Hook:
 """
Expand Down

1 change: 1 addition & 0 deletions
toltecmk

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
toltecmk.py

59 changes: 59 additions & 0 deletions
toltecmk-contained

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
# Copyright (c) 2021 The Toltec Contributors
# SPDX-License-Identifier: MIT

import argparse
import os
import subprocess
from pathlib import Path

from toltec import parse_recipe
from toltecmk import make_argparser

# Prefix for all Toltec Docker images
IMAGE_PREFIX = "ghcr.io/toltec-dev/"

args = make_argparser().parse_args()

recipe_bundle = parse_recipe(args.recipe_dir)
# Don't recipes in a bundle share most information?
recipe = next(iter(recipe_bundle.values()))

image = IMAGE_PREFIX + recipe.image

toltecmk_dir = str(Path(__file__).resolve().parent)

mounts = [ (args.recipe_dir, '/recipe'),
 (args.work_dir, '/build'),
 (args.dist_dir, '/pkg'),
 (toltecmk_dir, '/toltecmk'),
 ]

for m in mounts:
 if not os.path.exists(m[0]):
 os.makedirs(m[0])

if args.source_dir:
 mounts.append((args.source_dir, '/src'))

cmd = ['podman', 'run', '-it', '--rm']
for m in mounts:
 cmd.append('-v')
 cmd.append(':'.join(m))
cmd.append(image)
cmd += ['bash', '-c']
# currently required by toltecmk and missing in images
internal_cmd = []
internal_cmd += ['apt update && apt install -yq python3-dateutil python3-requests && ']
internal_cmd += ['/toltecmk/toltecmk',
 '-w', '/build',
 '-d', '/pkg',
 '/recipe']
if args.source_dir: internal_cmd += ['-s', '/src']
if args.arch_name: internal_cmd += ['-a', args.arch_name]
if args.package_name: internal_cmd += ['-p', args.package_name]

cmd.append(' '.join(internal_cmd))
print(' '.join('"' + c + '"' if ' ' in c else c for c in cmd))
subprocess.call(cmd)

Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content

This repository was archived by the owner on Aug 12, 2026. It is now read-only.

Removed dependency on docker#21
 Draft

Bulkin
wants to merge
3
commits into

base:main
Choose a base branch

from

 Draft


Removed dependency on docker
#21
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
a4bdab4

Removed dependency on docker, added `toltecmk-contained` script...


Bulkin Aug 12, 2021
0ec133d

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021
4ca6575

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 11 additions & 1 deletion
toltec/__main__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,16 @@ def main() -> int:
 (default: [current directory]/dist)""",
 )

 parser.add_argument(
 "-s",
 "--source-dir",
 metavar="DIR",
 default=None,
 help="""path to the source directory
 (optional: when specified, a local build is performed instead of fetching 
 sources)""",
 )

 parser.add_argument(
 "-a",
 "--arch-name",
Expand DownExpand Up@@ -80,7 +90,7 @@ def main() -> int:

 recipe_bundle = parse_recipe(args.recipe_dir)

 with Builder(args.work_dir, args.dist_dir) as builder:
 with Builder(args.work_dir, args.dist_dir, args.source_dir) as builder:
 if args.hook:
 for ident in args.hook:
 if ident and ident[0] in (".", "/"):
Expand Down

54 changes: 0 additions & 54 deletions
toltec/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,7 +8,6 @@
import logging
from collections import deque
from typing import Deque, Dict, Generator, List, Optional, Tuple, Union
from docker.client import DockerClient

AssociativeArray = Dict[str, str]
IndexedArray = List[Optional[str]]
Expand DownExpand Up@@ -363,59 +362,6 @@ def run_script(variables: Variables, script: str) -> LogGenerator:
 raise ScriptError(f"Script exited with code {process.returncode}")


def run_script_in_container(
 docker: DockerClient,
 image: str,
 mounts: List,
 variables: Variables,
 script: str,
) -> LogGenerator:
 """
 Run a Bash script inside a Docker container and stream its output.

 :param docker: Docker client
 :param image: image to use for the new container
 :param mounts: paths to mount in the container
 :param variables: Bash variables to set before running the script
 :param script: Bash script to execute
 :returns: generator yielding output lines from the script
 :raises ScriptError: if the script exits with a non-zero code
 """
 container = docker.containers.run(
 image,
 mounts=mounts,
 command=[
 "/usr/bin/env",
 "bash",
 "-c",
 "\n".join(
 (
 "set -euo pipefail",
 put_variables(variables),
 "script() {",
 script,
 "}",
 "script",
 )
 ),
 ],
 detach=True,
 security_opt=["label=disable"],
 )

 try:
 for line in container.logs(stream=True):
 if line:
 yield line.decode().strip()

 result = container.wait()

 if result["StatusCode"] != 0:
 raise ScriptError(f"Script exited with code {result['StatusCode']}")
 finally:
 container.remove()


def pipe_logs(
 logger: logging.Logger,
 logs: LogGenerator,
Expand Down

62 changes: 17 additions & 45 deletions
toltec/builder.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,7 +9,6 @@
import os
import logging
import textwrap
import docker
import requests
from . import bash, util, ipk
from .recipe import RecipeBundle, Recipe, Package
Expand All@@ -28,10 +27,9 @@ class Builder: # pylint: disable=too-few-public-methods
 # Detect non-local paths
 URL_REGEX = re.compile(r"[a-z]+://")

 # Prefix for all Toltec Docker images
 IMAGE_PREFIX = "ghcr.io/toltec-dev/"

 def __init__(self, work_dir: str, dist_dir: str) -> None:
 def __init__(
 self, work_dir: str, dist_dir: str, source_dir: Optional[str]
 ) -> None:
 """
 Create a builder helper.

Expand All@@ -40,15 +38,7 @@ def __init__(self, work_dir: str, dist_dir: str) -> None:
 """
 self.work_dir = work_dir
 self.dist_dir = dist_dir

 try:
 self.docker = docker.from_env()
 except docker.errors.DockerException as err:
 raise BuildError(
 "Unable to connect to the Docker daemon. \
Please check that the service is running and that you have the necessary \
permissions."
 ) from err
 self.source_dir = source_dir

 def __enter__(self) -> "Builder":
 return self
Expand All@@ -59,7 +49,7 @@ def __exit__(
 exc_value: Optional[BaseException],
 traceback: Optional[TracebackType],
 ) -> None:
self.docker.close()
pass

 @util.hook
 def post_parse(self, recipe: Recipe) -> None:
Expand DownExpand Up@@ -164,13 +154,13 @@ def _make_arch(
 packages: Optional[List[Package]] = None,
 ) -> bool:
 self.post_parse(recipe)

 src_dir = os.path.join(build_dir, "src")
os.makedirs(src_dir, exist_ok=True)

self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)

 if self.source_dir:
src_dir = self.source_dir
else:
 src_dir = os.path.join(build_dir, "src")
 os.makedirs(src_dir, exist_ok=True)
self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)
 self._prepare(recipe, src_dir)
 self.post_prepare(recipe, src_dir)

Expand DownExpand Up@@ -274,8 +264,6 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 for filename in util.list_tree(src_dir):
 os.utime(filename, (epoch, epoch))

 mount_src = "/src"
 repo_src = "/repo"
 uid = os.getuid()
 pre_script: List[str] = []

Expand DownExpand Up@@ -331,33 +319,17 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 " -- " + " ".join(host_deps),
 )
 )

 logs = bash.run_script_in_container(
 self.docker,
 image=self.IMAGE_PREFIX + recipe.image,
 mounts=[
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(src_dir),
 target=mount_src,
 ),
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(self.dist_dir),
 target=repo_src,
 ),
 ],
 variables={
 "srcdir": mount_src,
 },
 logs = bash.run_script(
 script="\n".join(
 (
 f'cd "{src_dir}"',
 *pre_script,
 f'cd "{mount_src}"',
 recipe.build,
 f'chown -R {uid}:{uid} "{mount_src}"',
 )
 ),
 variables={
 "srcdir": src_dir,
 },
 )
 bash.pipe_logs(logger, logs, "build()")

Expand Down

23 changes: 11 additions & 12 deletions
toltec/util.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,7 +19,7 @@
 IO,
 List,
 Optional,
 Protocol,
# Protocol,
 Type,
 Union,
)
Expand DownExpand Up@@ -302,6 +302,8 @@ def check_directory(path: str, message: str) -> bool:
 try:
 os.mkdir(path)
 except FileExistsError:
 if not os.listdir(path):
 return True
 ans = query_user(
 message,
 default="c",
Expand DownExpand Up@@ -343,18 +345,15 @@ def list_tree(root: str) -> List[str]:
HookTrigger = Callable[..., None]
HookListener = Callable[..., None]

# Protocol is not available in python 3.7 (Debian buster)
Hook = Any
# class Hook(Protocol): # pylint:disable=too-few-public-methods
# """Protocol for hooks."""

class Hook(Protocol): # pylint:disable=too-few-public-methods
 """Protocol for hooks."""

 @staticmethod
 def register(new_listener: HookListener) -> None:
 """Add a new listener to this hook."""
 ...

 # Invoke all listeners for this hook
 __call__: HookTrigger

# @staticmethod
# def register(new_listener: HookListener) -> None:
# """Add a new listener to this hook."""
# ...

def hook(func: HookTrigger) -> Hook:
 """
Expand Down

1 change: 1 addition & 0 deletions
toltecmk

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
toltecmk.py

59 changes: 59 additions & 0 deletions
toltecmk-contained

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
# Copyright (c) 2021 The Toltec Contributors
# SPDX-License-Identifier: MIT

import argparse
import os
import subprocess
from pathlib import Path

from toltec import parse_recipe
from toltecmk import make_argparser

# Prefix for all Toltec Docker images
IMAGE_PREFIX = "ghcr.io/toltec-dev/"

args = make_argparser().parse_args()

recipe_bundle = parse_recipe(args.recipe_dir)
# Don't recipes in a bundle share most information?
recipe = next(iter(recipe_bundle.values()))

image = IMAGE_PREFIX + recipe.image

toltecmk_dir = str(Path(__file__).resolve().parent)

mounts = [ (args.recipe_dir, '/recipe'),
 (args.work_dir, '/build'),
 (args.dist_dir, '/pkg'),
 (toltecmk_dir, '/toltecmk'),
 ]

for m in mounts:
 if not os.path.exists(m[0]):
 os.makedirs(m[0])

if args.source_dir:
 mounts.append((args.source_dir, '/src'))

cmd = ['podman', 'run', '-it', '--rm']
for m in mounts:
 cmd.append('-v')
 cmd.append(':'.join(m))
cmd.append(image)
cmd += ['bash', '-c']
# currently required by toltecmk and missing in images
internal_cmd = []
internal_cmd += ['apt update && apt install -yq python3-dateutil python3-requests && ']
internal_cmd += ['/toltecmk/toltecmk',
 '-w', '/build',
 '-d', '/pkg',
 '/recipe']
if args.source_dir: internal_cmd += ['-s', '/src']
if args.arch_name: internal_cmd += ['-a', args.arch_name]
if args.package_name: internal_cmd += ['-p', args.package_name]

cmd.append(' '.join(internal_cmd))
print(' '.join('"' + c + '"' if ' ' in c else c for c in cmd))
subprocess.call(cmd)

Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content

This repository was archived by the owner on Aug 12, 2026. It is now read-only.

Removed dependency on docker#21
 Draft

Bulkin
wants to merge
3
commits into

base:main
Choose a base branch

from

 Draft


Removed dependency on docker
#21
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
a4bdab4

Removed dependency on docker, added `toltecmk-contained` script...


Bulkin Aug 12, 2021
0ec133d

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021
4ca6575

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 11 additions & 1 deletion
toltec/__main__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,16 @@ def main() -> int:
 (default: [current directory]/dist)""",
 )

 parser.add_argument(
 "-s",
 "--source-dir",
 metavar="DIR",
 default=None,
 help="""path to the source directory
 (optional: when specified, a local build is performed instead of fetching 
 sources)""",
 )

 parser.add_argument(
 "-a",
 "--arch-name",
Expand DownExpand Up@@ -80,7 +90,7 @@ def main() -> int:

 recipe_bundle = parse_recipe(args.recipe_dir)

 with Builder(args.work_dir, args.dist_dir) as builder:
 with Builder(args.work_dir, args.dist_dir, args.source_dir) as builder:
 if args.hook:
 for ident in args.hook:
 if ident and ident[0] in (".", "/"):
Expand Down

54 changes: 0 additions & 54 deletions
toltec/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,7 +8,6 @@
import logging
from collections import deque
from typing import Deque, Dict, Generator, List, Optional, Tuple, Union
from docker.client import DockerClient

AssociativeArray = Dict[str, str]
IndexedArray = List[Optional[str]]
Expand DownExpand Up@@ -363,59 +362,6 @@ def run_script(variables: Variables, script: str) -> LogGenerator:
 raise ScriptError(f"Script exited with code {process.returncode}")


def run_script_in_container(
 docker: DockerClient,
 image: str,
 mounts: List,
 variables: Variables,
 script: str,
) -> LogGenerator:
 """
 Run a Bash script inside a Docker container and stream its output.

 :param docker: Docker client
 :param image: image to use for the new container
 :param mounts: paths to mount in the container
 :param variables: Bash variables to set before running the script
 :param script: Bash script to execute
 :returns: generator yielding output lines from the script
 :raises ScriptError: if the script exits with a non-zero code
 """
 container = docker.containers.run(
 image,
 mounts=mounts,
 command=[
 "/usr/bin/env",
 "bash",
 "-c",
 "\n".join(
 (
 "set -euo pipefail",
 put_variables(variables),
 "script() {",
 script,
 "}",
 "script",
 )
 ),
 ],
 detach=True,
 security_opt=["label=disable"],
 )

 try:
 for line in container.logs(stream=True):
 if line:
 yield line.decode().strip()

 result = container.wait()

 if result["StatusCode"] != 0:
 raise ScriptError(f"Script exited with code {result['StatusCode']}")
 finally:
 container.remove()


def pipe_logs(
 logger: logging.Logger,
 logs: LogGenerator,
Expand Down

62 changes: 17 additions & 45 deletions
toltec/builder.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,7 +9,6 @@
import os
import logging
import textwrap
import docker
import requests
from . import bash, util, ipk
from .recipe import RecipeBundle, Recipe, Package
Expand All@@ -28,10 +27,9 @@ class Builder: # pylint: disable=too-few-public-methods
 # Detect non-local paths
 URL_REGEX = re.compile(r"[a-z]+://")

 # Prefix for all Toltec Docker images
 IMAGE_PREFIX = "ghcr.io/toltec-dev/"

 def __init__(self, work_dir: str, dist_dir: str) -> None:
 def __init__(
 self, work_dir: str, dist_dir: str, source_dir: Optional[str]
 ) -> None:
 """
 Create a builder helper.

Expand All@@ -40,15 +38,7 @@ def __init__(self, work_dir: str, dist_dir: str) -> None:
 """
 self.work_dir = work_dir
 self.dist_dir = dist_dir

 try:
 self.docker = docker.from_env()
 except docker.errors.DockerException as err:
 raise BuildError(
 "Unable to connect to the Docker daemon. \
Please check that the service is running and that you have the necessary \
permissions."
 ) from err
 self.source_dir = source_dir

 def __enter__(self) -> "Builder":
 return self
Expand All@@ -59,7 +49,7 @@ def __exit__(
 exc_value: Optional[BaseException],
 traceback: Optional[TracebackType],
 ) -> None:
self.docker.close()
pass

 @util.hook
 def post_parse(self, recipe: Recipe) -> None:
Expand DownExpand Up@@ -164,13 +154,13 @@ def _make_arch(
 packages: Optional[List[Package]] = None,
 ) -> bool:
 self.post_parse(recipe)

 src_dir = os.path.join(build_dir, "src")
os.makedirs(src_dir, exist_ok=True)

self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)

 if self.source_dir:
src_dir = self.source_dir
else:
 src_dir = os.path.join(build_dir, "src")
 os.makedirs(src_dir, exist_ok=True)
self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)
 self._prepare(recipe, src_dir)
 self.post_prepare(recipe, src_dir)

Expand DownExpand Up@@ -274,8 +264,6 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 for filename in util.list_tree(src_dir):
 os.utime(filename, (epoch, epoch))

 mount_src = "/src"
 repo_src = "/repo"
 uid = os.getuid()
 pre_script: List[str] = []

Expand DownExpand Up@@ -331,33 +319,17 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 " -- " + " ".join(host_deps),
 )
 )

 logs = bash.run_script_in_container(
 self.docker,
 image=self.IMAGE_PREFIX + recipe.image,
 mounts=[
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(src_dir),
 target=mount_src,
 ),
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(self.dist_dir),
 target=repo_src,
 ),
 ],
 variables={
 "srcdir": mount_src,
 },
 logs = bash.run_script(
 script="\n".join(
 (
 f'cd "{src_dir}"',
 *pre_script,
 f'cd "{mount_src}"',
 recipe.build,
 f'chown -R {uid}:{uid} "{mount_src}"',
 )
 ),
 variables={
 "srcdir": src_dir,
 },
 )
 bash.pipe_logs(logger, logs, "build()")

Expand Down

23 changes: 11 additions & 12 deletions
toltec/util.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,7 +19,7 @@
 IO,
 List,
 Optional,
 Protocol,
# Protocol,
 Type,
 Union,
)
Expand DownExpand Up@@ -302,6 +302,8 @@ def check_directory(path: str, message: str) -> bool:
 try:
 os.mkdir(path)
 except FileExistsError:
 if not os.listdir(path):
 return True
 ans = query_user(
 message,
 default="c",
Expand DownExpand Up@@ -343,18 +345,15 @@ def list_tree(root: str) -> List[str]:
HookTrigger = Callable[..., None]
HookListener = Callable[..., None]

# Protocol is not available in python 3.7 (Debian buster)
Hook = Any
# class Hook(Protocol): # pylint:disable=too-few-public-methods
# """Protocol for hooks."""

class Hook(Protocol): # pylint:disable=too-few-public-methods
 """Protocol for hooks."""

 @staticmethod
 def register(new_listener: HookListener) -> None:
 """Add a new listener to this hook."""
 ...

 # Invoke all listeners for this hook
 __call__: HookTrigger

# @staticmethod
# def register(new_listener: HookListener) -> None:
# """Add a new listener to this hook."""
# ...

def hook(func: HookTrigger) -> Hook:
 """
Expand Down

1 change: 1 addition & 0 deletions
toltecmk

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
toltecmk.py

59 changes: 59 additions & 0 deletions
toltecmk-contained

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
# Copyright (c) 2021 The Toltec Contributors
# SPDX-License-Identifier: MIT

import argparse
import os
import subprocess
from pathlib import Path

from toltec import parse_recipe
from toltecmk import make_argparser

# Prefix for all Toltec Docker images
IMAGE_PREFIX = "ghcr.io/toltec-dev/"

args = make_argparser().parse_args()

recipe_bundle = parse_recipe(args.recipe_dir)
# Don't recipes in a bundle share most information?
recipe = next(iter(recipe_bundle.values()))

image = IMAGE_PREFIX + recipe.image

toltecmk_dir = str(Path(__file__).resolve().parent)

mounts = [ (args.recipe_dir, '/recipe'),
 (args.work_dir, '/build'),
 (args.dist_dir, '/pkg'),
 (toltecmk_dir, '/toltecmk'),
 ]

for m in mounts:
 if not os.path.exists(m[0]):
 os.makedirs(m[0])

if args.source_dir:
 mounts.append((args.source_dir, '/src'))

cmd = ['podman', 'run', '-it', '--rm']
for m in mounts:
 cmd.append('-v')
 cmd.append(':'.join(m))
cmd.append(image)
cmd += ['bash', '-c']
# currently required by toltecmk and missing in images
internal_cmd = []
internal_cmd += ['apt update && apt install -yq python3-dateutil python3-requests && ']
internal_cmd += ['/toltecmk/toltecmk',
 '-w', '/build',
 '-d', '/pkg',
 '/recipe']
if args.source_dir: internal_cmd += ['-s', '/src']
if args.arch_name: internal_cmd += ['-a', args.arch_name]
if args.package_name: internal_cmd += ['-p', args.package_name]

cmd.append(' '.join(internal_cmd))
print(' '.join('"' + c + '"' if ' ' in c else c for c in cmd))
subprocess.call(cmd)

Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content

This repository was archived by the owner on Aug 12, 2026. It is now read-only.

Removed dependency on docker#21
 Draft

Bulkin
wants to merge
3
commits into

base:main
Choose a base branch

from

 Draft


Removed dependency on docker
#21
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
a4bdab4

Removed dependency on docker, added `toltecmk-contained` script...


Bulkin Aug 12, 2021
0ec133d

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021
4ca6575

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 11 additions & 1 deletion
toltec/__main__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,16 @@ def main() -> int:
 (default: [current directory]/dist)""",
 )

 parser.add_argument(
 "-s",
 "--source-dir",
 metavar="DIR",
 default=None,
 help="""path to the source directory
 (optional: when specified, a local build is performed instead of fetching 
 sources)""",
 )

 parser.add_argument(
 "-a",
 "--arch-name",
Expand DownExpand Up@@ -80,7 +90,7 @@ def main() -> int:

 recipe_bundle = parse_recipe(args.recipe_dir)

 with Builder(args.work_dir, args.dist_dir) as builder:
 with Builder(args.work_dir, args.dist_dir, args.source_dir) as builder:
 if args.hook:
 for ident in args.hook:
 if ident and ident[0] in (".", "/"):
Expand Down

54 changes: 0 additions & 54 deletions
toltec/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,7 +8,6 @@
import logging
from collections import deque
from typing import Deque, Dict, Generator, List, Optional, Tuple, Union
from docker.client import DockerClient

AssociativeArray = Dict[str, str]
IndexedArray = List[Optional[str]]
Expand DownExpand Up@@ -363,59 +362,6 @@ def run_script(variables: Variables, script: str) -> LogGenerator:
 raise ScriptError(f"Script exited with code {process.returncode}")


def run_script_in_container(
 docker: DockerClient,
 image: str,
 mounts: List,
 variables: Variables,
 script: str,
) -> LogGenerator:
 """
 Run a Bash script inside a Docker container and stream its output.

 :param docker: Docker client
 :param image: image to use for the new container
 :param mounts: paths to mount in the container
 :param variables: Bash variables to set before running the script
 :param script: Bash script to execute
 :returns: generator yielding output lines from the script
 :raises ScriptError: if the script exits with a non-zero code
 """
 container = docker.containers.run(
 image,
 mounts=mounts,
 command=[
 "/usr/bin/env",
 "bash",
 "-c",
 "\n".join(
 (
 "set -euo pipefail",
 put_variables(variables),
 "script() {",
 script,
 "}",
 "script",
 )
 ),
 ],
 detach=True,
 security_opt=["label=disable"],
 )

 try:
 for line in container.logs(stream=True):
 if line:
 yield line.decode().strip()

 result = container.wait()

 if result["StatusCode"] != 0:
 raise ScriptError(f"Script exited with code {result['StatusCode']}")
 finally:
 container.remove()


def pipe_logs(
 logger: logging.Logger,
 logs: LogGenerator,
Expand Down

62 changes: 17 additions & 45 deletions
toltec/builder.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,7 +9,6 @@
import os
import logging
import textwrap
import docker
import requests
from . import bash, util, ipk
from .recipe import RecipeBundle, Recipe, Package
Expand All@@ -28,10 +27,9 @@ class Builder: # pylint: disable=too-few-public-methods
 # Detect non-local paths
 URL_REGEX = re.compile(r"[a-z]+://")

 # Prefix for all Toltec Docker images
 IMAGE_PREFIX = "ghcr.io/toltec-dev/"

 def __init__(self, work_dir: str, dist_dir: str) -> None:
 def __init__(
 self, work_dir: str, dist_dir: str, source_dir: Optional[str]
 ) -> None:
 """
 Create a builder helper.

Expand All@@ -40,15 +38,7 @@ def __init__(self, work_dir: str, dist_dir: str) -> None:
 """
 self.work_dir = work_dir
 self.dist_dir = dist_dir

 try:
 self.docker = docker.from_env()
 except docker.errors.DockerException as err:
 raise BuildError(
 "Unable to connect to the Docker daemon. \
Please check that the service is running and that you have the necessary \
permissions."
 ) from err
 self.source_dir = source_dir

 def __enter__(self) -> "Builder":
 return self
Expand All@@ -59,7 +49,7 @@ def __exit__(
 exc_value: Optional[BaseException],
 traceback: Optional[TracebackType],
 ) -> None:
self.docker.close()
pass

 @util.hook
 def post_parse(self, recipe: Recipe) -> None:
Expand DownExpand Up@@ -164,13 +154,13 @@ def _make_arch(
 packages: Optional[List[Package]] = None,
 ) -> bool:
 self.post_parse(recipe)

 src_dir = os.path.join(build_dir, "src")
os.makedirs(src_dir, exist_ok=True)

self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)

 if self.source_dir:
src_dir = self.source_dir
else:
 src_dir = os.path.join(build_dir, "src")
 os.makedirs(src_dir, exist_ok=True)
self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)
 self._prepare(recipe, src_dir)
 self.post_prepare(recipe, src_dir)

Expand DownExpand Up@@ -274,8 +264,6 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 for filename in util.list_tree(src_dir):
 os.utime(filename, (epoch, epoch))

 mount_src = "/src"
 repo_src = "/repo"
 uid = os.getuid()
 pre_script: List[str] = []

Expand DownExpand Up@@ -331,33 +319,17 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 " -- " + " ".join(host_deps),
 )
 )

 logs = bash.run_script_in_container(
 self.docker,
 image=self.IMAGE_PREFIX + recipe.image,
 mounts=[
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(src_dir),
 target=mount_src,
 ),
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(self.dist_dir),
 target=repo_src,
 ),
 ],
 variables={
 "srcdir": mount_src,
 },
 logs = bash.run_script(
 script="\n".join(
 (
 f'cd "{src_dir}"',
 *pre_script,
 f'cd "{mount_src}"',
 recipe.build,
 f'chown -R {uid}:{uid} "{mount_src}"',
 )
 ),
 variables={
 "srcdir": src_dir,
 },
 )
 bash.pipe_logs(logger, logs, "build()")

Expand Down

23 changes: 11 additions & 12 deletions
toltec/util.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,7 +19,7 @@
 IO,
 List,
 Optional,
 Protocol,
# Protocol,
 Type,
 Union,
)
Expand DownExpand Up@@ -302,6 +302,8 @@ def check_directory(path: str, message: str) -> bool:
 try:
 os.mkdir(path)
 except FileExistsError:
 if not os.listdir(path):
 return True
 ans = query_user(
 message,
 default="c",
Expand DownExpand Up@@ -343,18 +345,15 @@ def list_tree(root: str) -> List[str]:
HookTrigger = Callable[..., None]
HookListener = Callable[..., None]

# Protocol is not available in python 3.7 (Debian buster)
Hook = Any
# class Hook(Protocol): # pylint:disable=too-few-public-methods
# """Protocol for hooks."""

class Hook(Protocol): # pylint:disable=too-few-public-methods
 """Protocol for hooks."""

 @staticmethod
 def register(new_listener: HookListener) -> None:
 """Add a new listener to this hook."""
 ...

 # Invoke all listeners for this hook
 __call__: HookTrigger

# @staticmethod
# def register(new_listener: HookListener) -> None:
# """Add a new listener to this hook."""
# ...

def hook(func: HookTrigger) -> Hook:
 """
Expand Down

1 change: 1 addition & 0 deletions
toltecmk

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
toltecmk.py

59 changes: 59 additions & 0 deletions
toltecmk-contained

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
# Copyright (c) 2021 The Toltec Contributors
# SPDX-License-Identifier: MIT

import argparse
import os
import subprocess
from pathlib import Path

from toltec import parse_recipe
from toltecmk import make_argparser

# Prefix for all Toltec Docker images
IMAGE_PREFIX = "ghcr.io/toltec-dev/"

args = make_argparser().parse_args()

recipe_bundle = parse_recipe(args.recipe_dir)
# Don't recipes in a bundle share most information?
recipe = next(iter(recipe_bundle.values()))

image = IMAGE_PREFIX + recipe.image

toltecmk_dir = str(Path(__file__).resolve().parent)

mounts = [ (args.recipe_dir, '/recipe'),
 (args.work_dir, '/build'),
 (args.dist_dir, '/pkg'),
 (toltecmk_dir, '/toltecmk'),
 ]

for m in mounts:
 if not os.path.exists(m[0]):
 os.makedirs(m[0])

if args.source_dir:
 mounts.append((args.source_dir, '/src'))

cmd = ['podman', 'run', '-it', '--rm']
for m in mounts:
 cmd.append('-v')
 cmd.append(':'.join(m))
cmd.append(image)
cmd += ['bash', '-c']
# currently required by toltecmk and missing in images
internal_cmd = []
internal_cmd += ['apt update && apt install -yq python3-dateutil python3-requests && ']
internal_cmd += ['/toltecmk/toltecmk',
 '-w', '/build',
 '-d', '/pkg',
 '/recipe']
if args.source_dir: internal_cmd += ['-s', '/src']
if args.arch_name: internal_cmd += ['-a', args.arch_name]
if args.package_name: internal_cmd += ['-p', args.package_name]

cmd.append(' '.join(internal_cmd))
print(' '.join('"' + c + '"' if ' ' in c else c for c in cmd))
subprocess.call(cmd)

Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content

This repository was archived by the owner on Aug 12, 2026. It is now read-only.

Removed dependency on docker#21
 Draft

Bulkin
wants to merge
3
commits into

base:main
Choose a base branch

from

 Draft


Removed dependency on docker
#21
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
a4bdab4

Removed dependency on docker, added `toltecmk-contained` script...


Bulkin Aug 12, 2021
0ec133d

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021
4ca6575

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 11 additions & 1 deletion
toltec/__main__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,16 @@ def main() -> int:
 (default: [current directory]/dist)""",
 )

 parser.add_argument(
 "-s",
 "--source-dir",
 metavar="DIR",
 default=None,
 help="""path to the source directory
 (optional: when specified, a local build is performed instead of fetching 
 sources)""",
 )

 parser.add_argument(
 "-a",
 "--arch-name",
Expand DownExpand Up@@ -80,7 +90,7 @@ def main() -> int:

 recipe_bundle = parse_recipe(args.recipe_dir)

 with Builder(args.work_dir, args.dist_dir) as builder:
 with Builder(args.work_dir, args.dist_dir, args.source_dir) as builder:
 if args.hook:
 for ident in args.hook:
 if ident and ident[0] in (".", "/"):
Expand Down

54 changes: 0 additions & 54 deletions
toltec/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,7 +8,6 @@
import logging
from collections import deque
from typing import Deque, Dict, Generator, List, Optional, Tuple, Union
from docker.client import DockerClient

AssociativeArray = Dict[str, str]
IndexedArray = List[Optional[str]]
Expand DownExpand Up@@ -363,59 +362,6 @@ def run_script(variables: Variables, script: str) -> LogGenerator:
 raise ScriptError(f"Script exited with code {process.returncode}")


def run_script_in_container(
 docker: DockerClient,
 image: str,
 mounts: List,
 variables: Variables,
 script: str,
) -> LogGenerator:
 """
 Run a Bash script inside a Docker container and stream its output.

 :param docker: Docker client
 :param image: image to use for the new container
 :param mounts: paths to mount in the container
 :param variables: Bash variables to set before running the script
 :param script: Bash script to execute
 :returns: generator yielding output lines from the script
 :raises ScriptError: if the script exits with a non-zero code
 """
 container = docker.containers.run(
 image,
 mounts=mounts,
 command=[
 "/usr/bin/env",
 "bash",
 "-c",
 "\n".join(
 (
 "set -euo pipefail",
 put_variables(variables),
 "script() {",
 script,
 "}",
 "script",
 )
 ),
 ],
 detach=True,
 security_opt=["label=disable"],
 )

 try:
 for line in container.logs(stream=True):
 if line:
 yield line.decode().strip()

 result = container.wait()

 if result["StatusCode"] != 0:
 raise ScriptError(f"Script exited with code {result['StatusCode']}")
 finally:
 container.remove()


def pipe_logs(
 logger: logging.Logger,
 logs: LogGenerator,
Expand Down

62 changes: 17 additions & 45 deletions
toltec/builder.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,7 +9,6 @@
import os
import logging
import textwrap
import docker
import requests
from . import bash, util, ipk
from .recipe import RecipeBundle, Recipe, Package
Expand All@@ -28,10 +27,9 @@ class Builder: # pylint: disable=too-few-public-methods
 # Detect non-local paths
 URL_REGEX = re.compile(r"[a-z]+://")

 # Prefix for all Toltec Docker images
 IMAGE_PREFIX = "ghcr.io/toltec-dev/"

 def __init__(self, work_dir: str, dist_dir: str) -> None:
 def __init__(
 self, work_dir: str, dist_dir: str, source_dir: Optional[str]
 ) -> None:
 """
 Create a builder helper.

Expand All@@ -40,15 +38,7 @@ def __init__(self, work_dir: str, dist_dir: str) -> None:
 """
 self.work_dir = work_dir
 self.dist_dir = dist_dir

 try:
 self.docker = docker.from_env()
 except docker.errors.DockerException as err:
 raise BuildError(
 "Unable to connect to the Docker daemon. \
Please check that the service is running and that you have the necessary \
permissions."
 ) from err
 self.source_dir = source_dir

 def __enter__(self) -> "Builder":
 return self
Expand All@@ -59,7 +49,7 @@ def __exit__(
 exc_value: Optional[BaseException],
 traceback: Optional[TracebackType],
 ) -> None:
self.docker.close()
pass

 @util.hook
 def post_parse(self, recipe: Recipe) -> None:
Expand DownExpand Up@@ -164,13 +154,13 @@ def _make_arch(
 packages: Optional[List[Package]] = None,
 ) -> bool:
 self.post_parse(recipe)

 src_dir = os.path.join(build_dir, "src")
os.makedirs(src_dir, exist_ok=True)

self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)

 if self.source_dir:
src_dir = self.source_dir
else:
 src_dir = os.path.join(build_dir, "src")
 os.makedirs(src_dir, exist_ok=True)
self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)
 self._prepare(recipe, src_dir)
 self.post_prepare(recipe, src_dir)

Expand DownExpand Up@@ -274,8 +264,6 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 for filename in util.list_tree(src_dir):
 os.utime(filename, (epoch, epoch))

 mount_src = "/src"
 repo_src = "/repo"
 uid = os.getuid()
 pre_script: List[str] = []

Expand DownExpand Up@@ -331,33 +319,17 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 " -- " + " ".join(host_deps),
 )
 )

 logs = bash.run_script_in_container(
 self.docker,
 image=self.IMAGE_PREFIX + recipe.image,
 mounts=[
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(src_dir),
 target=mount_src,
 ),
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(self.dist_dir),
 target=repo_src,
 ),
 ],
 variables={
 "srcdir": mount_src,
 },
 logs = bash.run_script(
 script="\n".join(
 (
 f'cd "{src_dir}"',
 *pre_script,
 f'cd "{mount_src}"',
 recipe.build,
 f'chown -R {uid}:{uid} "{mount_src}"',
 )
 ),
 variables={
 "srcdir": src_dir,
 },
 )
 bash.pipe_logs(logger, logs, "build()")

Expand Down

23 changes: 11 additions & 12 deletions
toltec/util.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,7 +19,7 @@
 IO,
 List,
 Optional,
 Protocol,
# Protocol,
 Type,
 Union,
)
Expand DownExpand Up@@ -302,6 +302,8 @@ def check_directory(path: str, message: str) -> bool:
 try:
 os.mkdir(path)
 except FileExistsError:
 if not os.listdir(path):
 return True
 ans = query_user(
 message,
 default="c",
Expand DownExpand Up@@ -343,18 +345,15 @@ def list_tree(root: str) -> List[str]:
HookTrigger = Callable[..., None]
HookListener = Callable[..., None]

# Protocol is not available in python 3.7 (Debian buster)
Hook = Any
# class Hook(Protocol): # pylint:disable=too-few-public-methods
# """Protocol for hooks."""

class Hook(Protocol): # pylint:disable=too-few-public-methods
 """Protocol for hooks."""

 @staticmethod
 def register(new_listener: HookListener) -> None:
 """Add a new listener to this hook."""
 ...

 # Invoke all listeners for this hook
 __call__: HookTrigger

# @staticmethod
# def register(new_listener: HookListener) -> None:
# """Add a new listener to this hook."""
# ...

def hook(func: HookTrigger) -> Hook:
 """
Expand Down

1 change: 1 addition & 0 deletions
toltecmk

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
toltecmk.py

59 changes: 59 additions & 0 deletions
toltecmk-contained

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
# Copyright (c) 2021 The Toltec Contributors
# SPDX-License-Identifier: MIT

import argparse
import os
import subprocess
from pathlib import Path

from toltec import parse_recipe
from toltecmk import make_argparser

# Prefix for all Toltec Docker images
IMAGE_PREFIX = "ghcr.io/toltec-dev/"

args = make_argparser().parse_args()

recipe_bundle = parse_recipe(args.recipe_dir)
# Don't recipes in a bundle share most information?
recipe = next(iter(recipe_bundle.values()))

image = IMAGE_PREFIX + recipe.image

toltecmk_dir = str(Path(__file__).resolve().parent)

mounts = [ (args.recipe_dir, '/recipe'),
 (args.work_dir, '/build'),
 (args.dist_dir, '/pkg'),
 (toltecmk_dir, '/toltecmk'),
 ]

for m in mounts:
 if not os.path.exists(m[0]):
 os.makedirs(m[0])

if args.source_dir:
 mounts.append((args.source_dir, '/src'))

cmd = ['podman', 'run', '-it', '--rm']
for m in mounts:
 cmd.append('-v')
 cmd.append(':'.join(m))
cmd.append(image)
cmd += ['bash', '-c']
# currently required by toltecmk and missing in images
internal_cmd = []
internal_cmd += ['apt update && apt install -yq python3-dateutil python3-requests && ']
internal_cmd += ['/toltecmk/toltecmk',
 '-w', '/build',
 '-d', '/pkg',
 '/recipe']
if args.source_dir: internal_cmd += ['-s', '/src']
if args.arch_name: internal_cmd += ['-a', args.arch_name]
if args.package_name: internal_cmd += ['-p', args.package_name]

cmd.append(' '.join(internal_cmd))
print(' '.join('"' + c + '"' if ' ' in c else c for c in cmd))
subprocess.call(cmd)

Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content

This repository was archived by the owner on Aug 12, 2026. It is now read-only.

Removed dependency on docker#21
 Draft

Bulkin
wants to merge
3
commits into

base:main
Choose a base branch

from

 Draft


Removed dependency on docker
#21
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
a4bdab4

Removed dependency on docker, added `toltecmk-contained` script...


Bulkin Aug 12, 2021
0ec133d

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021
4ca6575

Merge branch 'main' into no-docker


matteodelabre Sep 1, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 11 additions & 1 deletion
toltec/__main__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,16 @@ def main() -> int:
 (default: [current directory]/dist)""",
 )

 parser.add_argument(
 "-s",
 "--source-dir",
 metavar="DIR",
 default=None,
 help="""path to the source directory
 (optional: when specified, a local build is performed instead of fetching 
 sources)""",
 )

 parser.add_argument(
 "-a",
 "--arch-name",
Expand DownExpand Up@@ -80,7 +90,7 @@ def main() -> int:

 recipe_bundle = parse_recipe(args.recipe_dir)

 with Builder(args.work_dir, args.dist_dir) as builder:
 with Builder(args.work_dir, args.dist_dir, args.source_dir) as builder:
 if args.hook:
 for ident in args.hook:
 if ident and ident[0] in (".", "/"):
Expand Down

54 changes: 0 additions & 54 deletions
toltec/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,7 +8,6 @@
import logging
from collections import deque
from typing import Deque, Dict, Generator, List, Optional, Tuple, Union
from docker.client import DockerClient

AssociativeArray = Dict[str, str]
IndexedArray = List[Optional[str]]
Expand DownExpand Up@@ -363,59 +362,6 @@ def run_script(variables: Variables, script: str) -> LogGenerator:
 raise ScriptError(f"Script exited with code {process.returncode}")


def run_script_in_container(
 docker: DockerClient,
 image: str,
 mounts: List,
 variables: Variables,
 script: str,
) -> LogGenerator:
 """
 Run a Bash script inside a Docker container and stream its output.

 :param docker: Docker client
 :param image: image to use for the new container
 :param mounts: paths to mount in the container
 :param variables: Bash variables to set before running the script
 :param script: Bash script to execute
 :returns: generator yielding output lines from the script
 :raises ScriptError: if the script exits with a non-zero code
 """
 container = docker.containers.run(
 image,
 mounts=mounts,
 command=[
 "/usr/bin/env",
 "bash",
 "-c",
 "\n".join(
 (
 "set -euo pipefail",
 put_variables(variables),
 "script() {",
 script,
 "}",
 "script",
 )
 ),
 ],
 detach=True,
 security_opt=["label=disable"],
 )

 try:
 for line in container.logs(stream=True):
 if line:
 yield line.decode().strip()

 result = container.wait()

 if result["StatusCode"] != 0:
 raise ScriptError(f"Script exited with code {result['StatusCode']}")
 finally:
 container.remove()


def pipe_logs(
 logger: logging.Logger,
 logs: LogGenerator,
Expand Down

62 changes: 17 additions & 45 deletions
toltec/builder.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,7 +9,6 @@
import os
import logging
import textwrap
import docker
import requests
from . import bash, util, ipk
from .recipe import RecipeBundle, Recipe, Package
Expand All@@ -28,10 +27,9 @@ class Builder: # pylint: disable=too-few-public-methods
 # Detect non-local paths
 URL_REGEX = re.compile(r"[a-z]+://")

 # Prefix for all Toltec Docker images
 IMAGE_PREFIX = "ghcr.io/toltec-dev/"

 def __init__(self, work_dir: str, dist_dir: str) -> None:
 def __init__(
 self, work_dir: str, dist_dir: str, source_dir: Optional[str]
 ) -> None:
 """
 Create a builder helper.

Expand All@@ -40,15 +38,7 @@ def __init__(self, work_dir: str, dist_dir: str) -> None:
 """
 self.work_dir = work_dir
 self.dist_dir = dist_dir

 try:
 self.docker = docker.from_env()
 except docker.errors.DockerException as err:
 raise BuildError(
 "Unable to connect to the Docker daemon. \
Please check that the service is running and that you have the necessary \
permissions."
 ) from err
 self.source_dir = source_dir

 def __enter__(self) -> "Builder":
 return self
Expand All@@ -59,7 +49,7 @@ def __exit__(
 exc_value: Optional[BaseException],
 traceback: Optional[TracebackType],
 ) -> None:
self.docker.close()
pass

 @util.hook
 def post_parse(self, recipe: Recipe) -> None:
Expand DownExpand Up@@ -164,13 +154,13 @@ def _make_arch(
 packages: Optional[List[Package]] = None,
 ) -> bool:
 self.post_parse(recipe)

 src_dir = os.path.join(build_dir, "src")
os.makedirs(src_dir, exist_ok=True)

self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)

 if self.source_dir:
src_dir = self.source_dir
else:
 src_dir = os.path.join(build_dir, "src")
 os.makedirs(src_dir, exist_ok=True)
self._fetch_sources(recipe, src_dir)
 self.post_fetch_sources(recipe, src_dir)
 self._prepare(recipe, src_dir)
 self.post_prepare(recipe, src_dir)

Expand DownExpand Up@@ -274,8 +264,6 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 for filename in util.list_tree(src_dir):
 os.utime(filename, (epoch, epoch))

 mount_src = "/src"
 repo_src = "/repo"
 uid = os.getuid()
 pre_script: List[str] = []

Expand DownExpand Up@@ -331,33 +319,17 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 " -- " + " ".join(host_deps),
 )
 )

 logs = bash.run_script_in_container(
 self.docker,
 image=self.IMAGE_PREFIX + recipe.image,
 mounts=[
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(src_dir),
 target=mount_src,
 ),
 docker.types.Mount(
 type="bind",
 source=os.path.abspath(self.dist_dir),
 target=repo_src,
 ),
 ],
 variables={
 "srcdir": mount_src,
 },
 logs = bash.run_script(
 script="\n".join(
 (
 f'cd "{src_dir}"',
 *pre_script,
 f'cd "{mount_src}"',
 recipe.build,
 f'chown -R {uid}:{uid} "{mount_src}"',
 )
 ),
 variables={
 "srcdir": src_dir,
 },
 )
 bash.pipe_logs(logger, logs, "build()")

Expand Down

23 changes: 11 additions & 12 deletions
toltec/util.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,7 +19,7 @@
 IO,
 List,
 Optional,
 Protocol,
# Protocol,
 Type,
 Union,
)
Expand DownExpand Up@@ -302,6 +302,8 @@ def check_directory(path: str, message: str) -> bool:
 try:
 os.mkdir(path)
 except FileExistsError:
 if not os.listdir(path):
 return True
 ans = query_user(
 message,
 default="c",
Expand DownExpand Up@@ -343,18 +345,15 @@ def list_tree(root: str) -> List[str]:
HookTrigger = Callable[..., None]
HookListener = Callable[..., None]

# Protocol is not available in python 3.7 (Debian buster)
Hook = Any
# class Hook(Protocol): # pylint:disable=too-few-public-methods
# """Protocol for hooks."""

class Hook(Protocol): # pylint:disable=too-few-public-methods
 """Protocol for hooks."""

 @staticmethod
 def register(new_listener: HookListener) -> None:
 """Add a new listener to this hook."""
 ...

 # Invoke all listeners for this hook
 __call__: HookTrigger

# @staticmethod
# def register(new_listener: HookListener) -> None:
# """Add a new listener to this hook."""
# ...

def hook(func: HookTrigger) -> Hook:
 """
Expand Down

1 change: 1 addition & 0 deletions
toltecmk

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
toltecmk.py

59 changes: 59 additions & 0 deletions
toltecmk-contained

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
# Copyright (c) 2021 The Toltec Contributors
# SPDX-License-Identifier: MIT

import argparse
import os
import subprocess
from pathlib import Path

from toltec import parse_recipe
from toltecmk import make_argparser

# Prefix for all Toltec Docker images
IMAGE_PREFIX = "ghcr.io/toltec-dev/"

args = make_argparser().parse_args()

recipe_bundle = parse_recipe(args.recipe_dir)
# Don't recipes in a bundle share most information?
recipe = next(iter(recipe_bundle.values()))

image = IMAGE_PREFIX + recipe.image

toltecmk_dir = str(Path(__file__).resolve().parent)

mounts = [ (args.recipe_dir, '/recipe'),
 (args.work_dir, '/build'),
 (args.dist_dir, '/pkg'),
 (toltecmk_dir, '/toltecmk'),
 ]

for m in mounts:
 if not os.path.exists(m[0]):
 os.makedirs(m[0])

if args.source_dir:
 mounts.append((args.source_dir, '/src'))

cmd = ['podman', 'run', '-it', '--rm']
for m in mounts:
 cmd.append('-v')
 cmd.append(':'.join(m))
cmd.append(image)
cmd += ['bash', '-c']
# currently required by toltecmk and missing in images
internal_cmd = []
internal_cmd += ['apt update && apt install -yq python3-dateutil python3-requests && ']
internal_cmd += ['/toltecmk/toltecmk',
 '-w', '/build',
 '-d', '/pkg',
 '/recipe']
if args.source_dir: internal_cmd += ['-s', '/src']
if args.arch_name: internal_cmd += ['-a', args.arch_name]
if args.package_name: internal_cmd += ['-p', args.package_name]

cmd.append(' '.join(internal_cmd))
print(' '.join('"' + c + '"' if ' ' in c else c for c in cmd))
subprocess.call(cmd)

Loading