diff --git a/dockerfiles/aws/test.dockerfile b/dockerfiles/aws/test.dockerfile index 81d6fd7..f299cce 100644 --- a/dockerfiles/aws/test.dockerfile +++ b/dockerfiles/aws/test.dockerfile @@ -15,6 +15,7 @@ WORKDIR /app COPY src/kernel_ci_cloud_labs/__init__.py /app/kernel_ci_cloud_labs/__init__.py COPY src/kernel_ci_cloud_labs/core/__init__.py /app/kernel_ci_cloud_labs/core/__init__.py COPY src/kernel_ci_cloud_labs/core/log_scrub.py /app/kernel_ci_cloud_labs/core/log_scrub.py +COPY src/kernel_ci_cloud_labs/kernel_test_results.py /app/kernel_ci_cloud_labs/kernel_test_results.py COPY src/kernel_ci_cloud_labs/launch_vm.py /app/launch_vm.py COPY src/kernel_ci_cloud_labs/debug_aws_setup.py /app/debug_aws_setup.py diff --git a/src/kernel_ci_cloud_labs/auth/aws_auth.py b/src/kernel_ci_cloud_labs/auth/aws_auth.py index b502260..fd00cf4 100644 --- a/src/kernel_ci_cloud_labs/auth/aws_auth.py +++ b/src/kernel_ci_cloud_labs/auth/aws_auth.py @@ -289,7 +289,8 @@ def _make_client(service): logger.debug("Execution role ARN: %s", task_config.get("execution_role_arn")) task_manager = AWSTaskDefinitionManager(ecs_client, {}) - task_manager.ensure_exists(task_config["family"], task_config) + task_arn, _ = task_manager.ensure_exists(task_config["family"], task_config) + self.config["ecs"]["task_definition"]["arn"] = task_arn # Store network manager for later use from kernel_ci_cloud_labs.auth.aws_network_manager import ( @@ -401,12 +402,7 @@ def _build_and_push_docker_image(self, repo_uri: str) -> str: try: ecr_client.describe_images(repositoryName=repo_name, imageIds=[{"imageTag": image_tag}]) logger.info("✓ Image already exists: %s", image_uri) - - # Update task definition config with existing image - if self.config.get("ecs", {}).get("task_definition"): - self.config["ecs"]["task_definition"]["image"] = image_uri - - return image_uri + return self._use_image_digest(ecr_client, repo_name, repo_uri, image_tag) except ecr_client.exceptions.ImageNotFoundException: logger.info("Image not found, building from %s", dockerfile) else: @@ -450,9 +446,20 @@ def _build_and_push_docker_image(self, repo_uri: str) -> str: subprocess.run(["docker", "push", image_uri], check=True) logger.info("✓ Image pushed: %s", image_uri) + return self._use_image_digest(ecr_client, repo_name, repo_uri, image_tag) + + def _use_image_digest(self, ecr_client, repo_name: str, repo_uri: str, image_tag: str) -> str: + """Point the task definition at the tag's current immutable digest. - # Update task definition config with new image + A mutable tag such as ``latest`` would let ECS start whatever image + the tag names at launch time, and the task-definition drift check + could never see the runner change. A digest reference changes with + the image, so a new runner registers a new revision. + """ + details = ecr_client.describe_images(repositoryName=repo_name, imageIds=[{"imageTag": image_tag}]) + digest = details["imageDetails"][0]["imageDigest"] + image_uri = f"{repo_uri}@{digest}" + logger.info("Runner image pinned: %s:%s -> %s", repo_uri, image_tag, image_uri) if self.config.get("ecs", {}).get("task_definition"): self.config["ecs"]["task_definition"]["image"] = image_uri - return image_uri diff --git a/src/kernel_ci_cloud_labs/auth/aws_task_definition_manager.py b/src/kernel_ci_cloud_labs/auth/aws_task_definition_manager.py index be883d8..7a9593f 100644 --- a/src/kernel_ci_cloud_labs/auth/aws_task_definition_manager.py +++ b/src/kernel_ci_cloud_labs/auth/aws_task_definition_manager.py @@ -6,13 +6,60 @@ from typing import Any, Dict +from botocore.exceptions import ClientError from kernel_ci_cloud_labs.core.base_resource_manager import BaseResourceManager +def _contains(actual, wanted): + """True when *actual* holds everything in *wanted*. + + describe_task_definition normalises what was registered: it adds defaults + such as ``logConfiguration.secretOptions: []`` and empty container lists. + Only the fields this manager sets are compared, recursively, so those + additions do not register a new revision on every run. + """ + if isinstance(wanted, dict): + return isinstance(actual, dict) and all(_contains(actual.get(key), value) for key, value in wanted.items()) + if isinstance(wanted, list): + return ( + isinstance(actual, list) + and len(actual) == len(wanted) + and all(_contains(item, value) for item, value in zip(actual, wanted)) + ) + return actual == wanted + + class AWSTaskDefinitionManager(BaseResourceManager): """Manages ECS task definitions""" + def ensure_exists(self, resource_name, resource_config=None, force_recreate=False): + """Register a revision when the executable contract changes. + + Finding an ACTIVE family is insufficient: it may still reference the + previous runner image. Existing revisions/tasks are never deregistered. + """ + config = resource_config or self.config.get(resource_name, {}) + try: + current = self.client.describe_task_definition(taskDefinition=resource_name)["taskDefinition"] + except ClientError as error: + if error.response.get("Error", {}).get("Code") != "ClientException": + raise + return self.create(resource_name, config), True + if current.get("status") == "ACTIVE" and not force_recreate: + if not config: + return current["taskDefinitionArn"], False + desired = self._definition(resource_name, config) + fields = ("cpu", "memory", "networkMode", "executionRoleArn", "taskRoleArn") + containers = current.get("containerDefinitions", []) + expected = desired["containerDefinitions"] + matches = all(current.get(key) == desired.get(key) for key in fields) + matches = matches and len(containers) == len(expected) + matches = matches and all(_contains(actual, wanted) for actual, wanted in zip(containers, expected)) + if matches: + return current["taskDefinitionArn"], False + return self.create(resource_name, config), True + def check_exists(self, resource_name: str) -> bool: """Check if task definition exists""" try: @@ -25,6 +72,12 @@ def check_exists(self, resource_name: str) -> bool: def create(self, resource_name: str, resource_config: Dict[str, Any]) -> str: """Create task definition with CloudWatch logs""" + response = self.client.register_task_definition(**self._definition(resource_name, resource_config)) + return response["taskDefinition"]["taskDefinitionArn"] + + @staticmethod + def _definition(resource_name, resource_config): + # Build container definition container_def = { "name": resource_config.get("container_name", "app"), @@ -50,7 +103,7 @@ def create(self, resource_name: str, resource_config: Dict[str, Any]) -> str: } # Register task definition - response = self.client.register_task_definition( + return dict( family=resource_name, networkMode="awsvpc", requiresCompatibilities=["FARGATE"], @@ -61,8 +114,6 @@ def create(self, resource_name: str, resource_config: Dict[str, Any]) -> str: containerDefinitions=[container_def], ) - return response["taskDefinition"]["taskDefinitionArn"] - def get_identifier(self, resource_name: str) -> str: """Get task definition ARN""" response = self.client.describe_task_definition(taskDefinition=resource_name) diff --git a/src/kernel_ci_cloud_labs/core/pipeline.py b/src/kernel_ci_cloud_labs/core/pipeline.py index efcba23..f3de790 100644 --- a/src/kernel_ci_cloud_labs/core/pipeline.py +++ b/src/kernel_ci_cloud_labs/core/pipeline.py @@ -13,13 +13,41 @@ from kernel_ci_cloud_labs.auth.aws_cloudwatch_manager import AWSCloudWatchManager from kernel_ci_cloud_labs.core.artifacts import collect_run_artifacts from kernel_ci_cloud_labs.core.logging_config import create_run_directory, get_logger +from kernel_ci_cloud_labs.kernel_test_results import EVIDENCE_MARKER, requires_evidence logger = get_logger(__name__) -def parse_vm_logs(vms_dir): +def parse_evidence_verdicts(container_log): + """Read per-instance native-test verdicts emitted by launch_vm.py. + + Returns ``{instance_id: verdict}``; a missing or unreadable log yields an + empty dict, which fails every evidence-requiring instance closed. + """ + verdicts = {} + try: + content = Path(container_log).read_text(encoding="utf-8") + except OSError: + return verdicts + for line in content.splitlines(): + if EVIDENCE_MARKER not in line: + continue + try: + verdict = json.loads(line.split(EVIDENCE_MARKER, 1)[1]) + except ValueError: + continue + if isinstance(verdict, dict) and verdict.get("instance_id"): + verdicts[verdict["instance_id"]] = verdict + return verdicts + + +def parse_vm_logs(vms_dir, evidence_tests=(), verdicts=None): """Parse VM log files and extract test statistics. + Instances of a test in *evidence_tests* pass only when *verdicts* (see + ``parse_evidence_verdicts``) holds an accepted verdict for that instance; + their entry also carries ``boot_status`` from the same verdict. + Expects one ``vms/.log`` per VM (written from CloudWatch streams). Boot-console logs written alongside as ``vms/-console.log`` by ``core.artifacts.collect_run_artifacts`` @@ -69,7 +97,14 @@ def parse_vm_logs(vms_dir): break # Check success/failure - if "Test execution completed: SUCCESS" in content: + passed = "Test execution completed: SUCCESS" in content + entry = {"test": test_name, "instance_id": instance_id} + if test_name in evidence_tests: + verdict = (verdicts or {}).get(instance_id) or {} + passed = passed and verdict.get("result") == "PASS" + if verdict.get("boot_requested"): + entry["boot_status"] = "PASS" if verdict.get("boot") == "PASS" else "FAIL" + if passed: stats["successful"] += 1 status = "PASS" else: @@ -79,9 +114,8 @@ def parse_vm_logs(vms_dir): stats["failed_tests"][test_name] = [] stats["failed_tests"][test_name].append(instance_id) - stats["instances"].append( - {"test": test_name, "instance_id": instance_id, "status": status} - ) + entry["status"] = status + stats["instances"].append(entry) except Exception as e: logger.warning("Could not parse VM log %s: %s", log_file, e) @@ -184,6 +218,7 @@ def create_summary( expected_vm_count=None, s3_context=None, container_failure_log_url=None, + evidence_tests=(), ): """Create summary.json with VM statistics. @@ -197,6 +232,8 @@ def create_summary( it exited non-zero before launching any VM. The KCIDB submitter uses this as ``tests[*].log_url`` for the synthetic Infrastructure row so users have a link to the actual failure reason. + evidence_tests: Test names whose VM success also requires a verified + evidence verdict in ``container.log`` (native pull-lab tests). """ end_time = time.time() total_runtime = end_time - start_time @@ -205,7 +242,8 @@ def create_summary( # Parse VM logs vms_dir = Path(run_dir) / "vms" - vm_stats = parse_vm_logs(vms_dir) + verdicts = parse_evidence_verdicts(Path(run_dir) / "container.log") if evidence_tests else {} + vm_stats = parse_vm_logs(vms_dir, evidence_tests=evidence_tests, verdicts=verdicts) # Determine status based on expected vs actual status = "success" @@ -350,6 +388,7 @@ def run_pipeline( test_id = None test_names = set() # Collect unique test names expected_vm_count = 0 # Calculate expected VM count + evidence_tests = set() # Tests that must also pass structured verification if hasattr(provider, "config") and provider.config: test_config = provider.config.get("test_config", {}) test_id = test_config.get("test_id") @@ -358,6 +397,8 @@ def run_pipeline( vms = test_config.get("vms", []) for vm in vms: test_value = vm.get("test") + if requires_evidence(vm.get("test_params")): + evidence_tests.update(test_value if isinstance(test_value, list) else [test_value]) # Handle list of tests if isinstance(test_value, list): for test_name in test_value: @@ -749,4 +790,5 @@ def run_pipeline( if "container_failure_log_url" in locals() else None ), + evidence_tests=evidence_tests if "evidence_tests" in locals() else (), ) diff --git a/src/kernel_ci_cloud_labs/kernel_test_results.py b/src/kernel_ci_cloud_labs/kernel_test_results.py new file mode 100644 index 0000000..17e1a49 --- /dev/null +++ b/src/kernel_ci_cloud_labs/kernel_test_results.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (C) 2026 Collabora Limited +"""Result contracts shared by VM payloads and the AWS-side verifier.""" + +import json +import re + +# A VM test whose parameters carry the pull-lab request list is a native +# kernel test: its VM-side SUCCESS only counts after the AWS-side verifier +# accepts its structured evidence. Keyed on the contract, not directory names. +TESTS_PARAM = "PULL_LABS_TESTS_JSON" +BOOT_TEST_TYPES = ("boot", "baseline") +# Container-log line prefix carrying one JSON verdict per verified instance. +EVIDENCE_MARKER = "PULLAB_EVIDENCE " + + +def requires_evidence(test_params): + return TESTS_PARAM in (test_params or {}) + + +def parse_runtest_names(text): + """Return test names from an LTP runtest file, as Kirk reads them. + + A runtest line is " "; blank lines and lines starting + with '#' are ignored. The command is never shell-parsed here. + """ + names = [] + for line in text.splitlines(): + line = line.strip() + if line and not line.startswith("#"): + names.append(line.split(None, 1)[0]) + return names + + +def validate_boot_evidence(data, kernel_url, arch, checksum=None): + if data.get("status") != "pass" or data.get("kernel_url") != kernel_url: + raise ValueError("Boot evidence does not identify the requested kernel artifact") + expected_arch = {"aarch64": "arm64"}.get(arch, arch) + if data.get("arch") != expected_arch: + raise ValueError("Boot evidence has the wrong architecture") + if not data.get("bootstrap_boot_id") or not data.get("boot_id") or data["boot_id"] == data["bootstrap_boot_id"]: + raise ValueError("Boot evidence does not show a reboot") + for key in ("release", "version"): + if not data.get(key) or data[key] != data.get("requested_" + key): + raise ValueError("Boot evidence has a different kernel " + key) + image_hash = data.get("image_sha256", "") + if not re.fullmatch(r"[a-f0-9]{64}", image_hash): + raise ValueError("Boot evidence has no valid image hash") + if "pullab.image_sha256=" + image_hash not in data.get("cmdline", "").split(): + raise ValueError("Boot evidence has no matching image marker") + if checksum and data.get("download_sha256") != checksum: + raise ValueError("Boot evidence does not match the requested artifact checksum") + + +def validate_kirk_report(report, expected_release, expected_tests=None, expected_version=None): + """Kirk exit status alone does not describe individual test outcomes.""" + rows = report.get("results") + if not isinstance(rows, list) or not rows: + raise ValueError("Kirk returned no individual LTP results") + reported_kernel = report.get("environment", {}).get("kernel") + identity_matches = reported_kernel == expected_release + if isinstance(reported_kernel, str) and reported_kernel.startswith("Linux "): + release, separator, version = reported_kernel[len("Linux ") :].partition(" ") + identity_matches = release == expected_release and bool(separator) and bool(version) + if expected_version is not None: + identity_matches = identity_matches and version == expected_version + if not identity_matches: + raise ValueError("LTP report was produced on a different kernel") + if expected_tests is not None: + if not expected_tests: + raise ValueError("No LTP tests were selected") + observed = {row.get("test_fqn") for row in rows} + missing = set(expected_tests) - observed + if missing: + raise ValueError("LTP did not report all selected tests: " + ", ".join(sorted(missing))) + passed = 0 + failed = [] + if any(int(report.get("stats", {}).get(key, 0)) for key in ("failed", "broken", "warnings")): + raise ValueError("Kirk aggregate statistics contain failed/broken/warning results") + for row in rows: + test = row.get("test") or {} + status = row.get("status") or test.get("result") + if test.get("result") and test["result"] != status: + raise ValueError("Kirk test status conflicts with its detailed result") + if status not in ("pass", "conf", "skip") or any( + int(test.get(key, 0)) for key in ("failed", "broken", "warnings") + ): + failed.append(row.get("test_fqn", "unknown")) + if status == "pass": + passed += 1 + if failed: + raise ValueError("LTP failures/broken tests: " + ", ".join(failed)) + if not passed: + raise ValueError("No LTP tests passed; empty/all-skipped runs are not passes") + return {"passed": passed, "skipped": len(rows) - passed, "failed": 0} + + +def verify_test_evidence(test_params, read_report): + """Check every requested native test of one VM against its evidence. + + *read_report(filename)* returns the parsed JSON artifact or raises. + Returns a verdict dict: "boot" and "result" are "PASS"/"FAIL" (boot is + judged separately so a later LTP failure cannot hide a good boot), and + "reason" carries the first rejection message. + """ + verdict = {"boot": "FAIL", "result": "FAIL", "boot_requested": False, "reason": ""} + try: + requests = json.loads(test_params[TESTS_PARAM]) + if not requests: + raise ValueError("No requested kernel tests in VM configuration") + verdict["boot_requested"] = any(request.get("type") in BOOT_TEST_TYPES for request in requests) + evidence = read_report("results_boot.json") + validate_boot_evidence( + evidence, + test_params["KERNEL_URL"], + test_params["ARCH"], + test_params.get("KERNEL_SHA256"), + ) + verdict["boot"] = "PASS" + for request in requests: + if request["type"] == "ltp": + ltp_report = read_report("results_ltp-" + request["id"] + ".json") + selection = ltp_report.get("pullab") or {} + if selection.get("requested_suites") != request["suites"]: + raise ValueError("LTP report does not identify the requested suites") + if selection.get("image_sha256") != evidence["image_sha256"]: + raise ValueError("LTP report has a different kernel image hash") + validate_kirk_report( + ltp_report, + evidence["release"], + selection.get("expected_tests", []), + expected_version=evidence["version"], + ) + elif request["type"] not in BOOT_TEST_TYPES: + raise ValueError("Unsupported native kernel test type: " + str(request["type"])) + verdict["result"] = "PASS" + except Exception as error: # pylint: disable=broad-exception-caught + verdict["reason"] = str(error) or type(error).__name__ + return verdict diff --git a/src/kernel_ci_cloud_labs/launch_vm.py b/src/kernel_ci_cloud_labs/launch_vm.py index ac4add2..e96ccce 100644 --- a/src/kernel_ci_cloud_labs/launch_vm.py +++ b/src/kernel_ci_cloud_labs/launch_vm.py @@ -19,6 +19,11 @@ from botocore.config import Config from kernel_ci_cloud_labs.core.log_scrub import scrub_text +from kernel_ci_cloud_labs.kernel_test_results import ( + EVIDENCE_MARKER, + requires_evidence, + verify_test_evidence, +) # Enable verbose diagnostics by setting KCI_DEBUG=1 (or true/yes/on) on the @@ -418,19 +423,54 @@ def check_test_result(self): content = obj["Body"].read().decode("utf-8").strip() log_not(f" result.txt: {content}") - if "SUCCESS" in content: - log_not("✓ Test result: SUCCESS") - return True - - log_error(f"✗ Test result: {content}") - return False - except self.s3.exceptions.NoSuchKey: + content = None log_error("✗ result.txt not found in S3") - return False except Exception as e: log_error(f"✗ Failed to read result.txt: {e}") - return False + return self._record_evidence(None) if requires_evidence(self.test_params) else False + + if requires_evidence(self.test_params): + return self._record_evidence(content) + if content is not None and "SUCCESS" in content: + log_not("✓ Test result: SUCCESS") + return True + if content is not None: + log_error(f"✗ Test result: {content}") + return False + + def _record_evidence(self, content): + """Verify a native kernel test's structured evidence and log the verdict. + + Evidence is checked even without a SUCCESS result so that a booted + kernel whose later tests failed still reports its boot outcome. The + marker line is the per-instance verdict the pipeline consumes. + """ + prefix = f"{self.run_prefix}/test_{self.test}/output/{self.instance_id}" + + def read_report(filename): + try: + artifact = self.s3.get_object(Bucket=self.s3_bucket, Key=f"{prefix}/{filename}") + except self.s3.exceptions.NoSuchKey as error: + raise ValueError(f"VM did not upload {filename}") from error + return json.loads(artifact["Body"].read()) + + verdict = verify_test_evidence(self.test_params, read_report) + if content != "SUCCESS": + verdict["result"] = "FAIL" + verdict["reason"] = verdict["reason"] or f"VM test result was {content!r}" + verdict["instance_id"] = self.instance_id + verdict["test"] = self.test + # The container log driver splits lines over 16 KiB; keep the verdict + # line parseable even when many LTP tests failed. + if verdict["reason"] and len(verdict["reason"]) > 2000: + verdict["reason"] = verdict["reason"][:2000] + "..." + print(EVIDENCE_MARKER + json.dumps(verdict, sort_keys=True), flush=True) + if verdict["result"] == "PASS": + log_not("✓ Test result: SUCCESS (structured evidence verified)") + return True + log_error(f"✗ Test evidence rejected (boot={verdict['boot']}): {verdict['reason']}") + return False def capture_console_output(self, reason="cleanup"): """Fetch EC2 serial console output (kernel boot log) and upload to S3. @@ -486,6 +526,11 @@ def capture_console_output(self, reason="cleanup"): InstanceId=self.instance_id, Latest=True ) output_b64 = resp.get("Output", "") + if not output_b64: + # A terminated Nitro VM can have no live serial buffer + # while its transition-triggered console snapshot exists. + resp = self.ec2.get_console_output(InstanceId=self.instance_id, Latest=False) + output_b64 = resp.get("Output", "") log_debug( f"get_console_output attempt={attempt} " f"output_len_b64={len(output_b64)} " diff --git a/src/kernel_ci_cloud_labs/providers/aws_provider.py b/src/kernel_ci_cloud_labs/providers/aws_provider.py index 97a0bbd..5cf76e6 100644 --- a/src/kernel_ci_cloud_labs/providers/aws_provider.py +++ b/src/kernel_ci_cloud_labs/providers/aws_provider.py @@ -77,6 +77,8 @@ def authenticate(self): logger.info("Using provided auth object to authenticate.") logger.debug("Calling auth.authenticate()") self.auth.authenticate() + task_config = self.config.get("ecs", {}).get("task_definition", {}) + self.task_definition = task_config.get("arn") or task_config.get("family", self.task_definition) logger.debug("Getting ECS client") self.ecs = self.auth.get_client("ecs") if not self.ecs: @@ -298,6 +300,13 @@ def _build_vm_log_manager(self, start_time_ms): ec2_log_group=ec2_log_group, ) + def _default_wait_timeout(self): + """Never stop the task before its own VMs' max_runtime can elapse.""" + vms = ((self.config or {}).get("test_config") or {}).get("vms") or [] + longest = max((int(vm.get("max_runtime") or 0) for vm in vms), default=0) + # Margin covers VM spawn, the kernel reboot, result upload and cleanup. + return max(3600, longest + 1800) + def wait_for_task_completion(self): """Wait for the ECS task to reach STOPPED, with crash / stall detection. @@ -315,7 +324,8 @@ def wait_for_task_completion(self): default accommodates CPU-heavy benchmarks (e.g. UnixBench) whose console goes quiet for many minutes during a run; lower it via the env var for faster hang detection on lighter workloads. - * Overall PULLAB_TASK_WAIT_TIMEOUT_SEC seconds elapsed (default 3600) + * Overall PULLAB_TASK_WAIT_TIMEOUT_SEC seconds elapsed (default: the + longest VM max_runtime plus spawn/cleanup margin, at least 3600) -- final safety net for whatever isn't covered above. Crash detection requires both cloudwatch.log_groups (with an /ec2/ @@ -335,7 +345,7 @@ def wait_for_task_completion(self): poll_interval = float(os.getenv("PULLAB_TASK_POLL_INTERVAL_SEC") or 30) log_interval = float(os.getenv("PULLAB_TASK_PROGRESS_LOG_SEC") or 120) hang_threshold = float(os.getenv("PULLAB_TASK_HANG_THRESHOLD_SEC") or 1200) - overall_timeout = float(os.getenv("PULLAB_TASK_WAIT_TIMEOUT_SEC") or 3600) + overall_timeout = float(os.getenv("PULLAB_TASK_WAIT_TIMEOUT_SEC") or self._default_wait_timeout()) start = time.time() start_ms = int(start * 1000) diff --git a/src/kernel_ci_cloud_labs/pull_labs_poller.py b/src/kernel_ci_cloud_labs/pull_labs_poller.py index 5057fe6..d19515b 100644 --- a/src/kernel_ci_cloud_labs/pull_labs_poller.py +++ b/src/kernel_ci_cloud_labs/pull_labs_poller.py @@ -463,6 +463,19 @@ def _extract_test_results(summary: Dict[str, Any]) -> Tuple[List[Dict[str, Any]] for inst in instances: test = inst.get("test", "unknown") instance_id = inst.get("instance_id", "") + # A native test VM that also served a requested boot/baseline test + # reports that boot on its own row, so a later LTP failure neither + # hides a good boot nor is a bad boot classified as a test failure. + if inst.get("boot_status") and _test_name_to_path(test) != "boot": + rows.append( + { + "name": "boot", + "status": inst["boot_status"], + "instance_id": instance_id, + "id_suffix": f"{instance_id}.boot", + "log_url": url_by_pair.get((test, instance_id)), + } + ) rows.append( { "name": _test_name_to_path(test), @@ -894,6 +907,8 @@ def _execute_job( build_id = f"{self.kcidb_origin}:unknown_{node_id}" try: + jobdef = dict(jobdef) + jobdef.setdefault("kernel_revision", (node.get("data") or {}).get("kernel_revision") or {}) run_config = translate_job(jobdef, self.base_config, node_id=node_id) except ValueError as e: logger.error("Translation failed for node %s: %s", node_id, e) @@ -924,7 +939,7 @@ def _execute_job( # KCIDB row is stable across retries (instead of positional `.{idx}`). test_rows = [] for idx, t in enumerate(per_test or []): - instance_suffix = t.get("instance_id") or str(idx) + instance_suffix = t.get("id_suffix") or t.get("instance_id") or str(idx) test_rows.append( build_test_row( origin=self.kcidb_origin, diff --git a/src/kernel_ci_cloud_labs/pull_labs_translate.py b/src/kernel_ci_cloud_labs/pull_labs_translate.py index 2695d21..9884f0a 100644 --- a/src/kernel_ci_cloud_labs/pull_labs_translate.py +++ b/src/kernel_ci_cloud_labs/pull_labs_translate.py @@ -16,9 +16,14 @@ """ import copy +import json +import re +import shlex import uuid from typing import Any, Dict, List, Optional +from kernel_ci_cloud_labs.kernel_test_results import BOOT_TEST_TYPES, TESTS_PARAM + # Architecture → instance_type / AMI hints. Override via the platform_map # argument of translate_job() to customise per-deployment. @@ -38,11 +43,11 @@ } # Map a PULL_LABS test "type" (jobdef.tests[].type) to a vm-tests directory. -# Extend via test_type_map argument; unknown types fall back to "url-kernel-boot". +# Unsupported tests must never silently become successful boot tests. DEFAULT_TEST_TYPE_MAP: Dict[str, str] = { + "boot": "url-kernel-boot", "baseline": "url-kernel-boot", - "ltp": "url-kernel-boot", - "unixbench": "url-kernel-boot", + "ltp": "url-kernel-ltp", } @@ -63,7 +68,82 @@ def _normalise_arch(jobdef: Dict[str, Any]) -> str: def _resolve_test_dir(test_type: str, test_type_map: Dict[str, str]) -> str: - return test_type_map.get(test_type, test_type_map.get("_default", "url-kernel-boot")) + directory = test_type_map.get(test_type) or test_type_map.get("_default") + if not directory: + raise ValueError(f"Unsupported pull-lab test type: {test_type!r}") + return directory + + +def _test_parameters(test: Dict[str, Any]) -> Dict[str, str]: + raw = test.get("parameters") or {} + if isinstance(raw, str): + parameters = {} + for field in shlex.split(raw): + key, separator, value = field.partition("=") + if not separator or key in parameters: + raise ValueError(f"Invalid or duplicate test parameter: {field!r}") + parameters[key] = value + return parameters + if not isinstance(raw, dict): + raise ValueError("Test parameters must be a mapping or key=value string") + return {key: str(value) for key, value in raw.items()} + + +def _test_requests(jobdef: Dict[str, Any], test_type_map: Dict[str, str]) -> List[Dict[str, Any]]: + """Validate the executable contract before claiming any test as supported.""" + tests = jobdef.get("tests") or [{"id": "boot", "type": "boot"}] + defaults = jobdef.get("parameters") or {} + requests = [] + seen = set() + for index, test in enumerate(tests): + kind = test.get("type", "") + _resolve_test_dir(kind, test_type_map) + name = test.get("id") or f"{kind}-{index}" + if not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or name in seen: + raise ValueError(f"Invalid or duplicate test id: {name!r}") + seen.add(name) + if test.get("pre-commands") or test.get("post-commands"): + raise ValueError("Pull-lab pre/post commands are not implemented by this runner") + timeout = int(test.get("timeout_s") or 600) + if not 0 < timeout <= 43200: + raise ValueError("Test timeout must be between 1 and 43200 seconds") + request = {"id": name, "type": kind, "timeout_s": timeout, "depends": list(test.get("depends") or [])} + params = _test_parameters(test) + if kind == "ltp": + unknown = set(params) - {"tst_cmdfiles", "skipfile", "workers", "skip_install"} + if unknown: + raise ValueError(f"Unsupported LTP parameters: {sorted(unknown)}") + if params.get("skip_install", "true").lower() not in ("true", "1"): + raise ValueError("LTP must be supplied by the job's rootfs artifact") + suites = params.get("tst_cmdfiles") or "smoketest" + suites = suites.split(",") + if any(not re.fullmatch(r"[A-Za-z0-9_.-]+", suite) for suite in suites): + raise ValueError("Invalid LTP suite name") + workers = params.get("workers", defaults.get("KIRK_WORKERS", "1")) + if str(workers) != "max" and not str(workers).isdigit(): + raise ValueError("LTP workers must be a positive integer or 'max'") + if str(workers).isdigit() and not 1 <= int(workers) <= 128: + raise ValueError("LTP workers must be between 1 and 128") + request.update( + suites=suites, + workers=str(workers), + skipfile=params.get("skipfile", defaults.get("SKIPFILE", "")), + runner=defaults.get("RUNNER", "/opt/kirk/kirk"), + ) + elif params: + raise ValueError(f"Parameters are not implemented for {kind!r}") + requests.append(request) + # Respect dependency order; reject unknown dependencies and cycles. + ordered = [] + pending = list(requests) + while pending: + ready = [request for request in pending if set(request["depends"]) <= {r["id"] for r in ordered}] + if not ready: + raise ValueError("Unknown or cyclic pull-lab test dependencies") + for request in ready: + ordered.append(request) + pending.remove(request) + return ordered def translate_job( @@ -86,11 +166,14 @@ def translate_job( to the vm-test as `test_params` so the run scripts can pick them up via env vars. """ - platform_map = platform_map or DEFAULT_PLATFORM_MAP + platform_map = platform_map or (base_config.get("kernelci") or {}).get("platform_map") or DEFAULT_PLATFORM_MAP test_type_map = test_type_map or DEFAULT_TEST_TYPE_MAP arch = _normalise_arch(jobdef) - plat_defaults = platform_map.get(arch) or platform_map["x86_64"] + platform_key = "arm64" if arch == "aarch64" and "arm64" in platform_map else arch + if platform_key not in platform_map: + raise ValueError(f"Unsupported pull-lab architecture: {arch!r}") + plat_defaults = platform_map[platform_key] artifacts = jobdef.get("artifacts", {}) or {} kernel_url = artifacts.get("kernel") @@ -99,11 +182,10 @@ def translate_job( if not kernel_url or not modules_url: raise ValueError( - "PULL_LABS job_definition missing required artifacts.kernel or " - "artifacts.modules — refusing to translate" + "PULL_LABS job_definition missing required artifacts.kernel or artifacts.modules — refusing to translate" ) - job_tests = jobdef.get("tests", []) or [] + job_tests = _test_requests(jobdef, test_type_map) test_dirs: List[str] = [] for t in job_tests: test_dirs.append(_resolve_test_dir(t.get("type", ""), test_type_map)) @@ -112,12 +194,38 @@ def translate_job( # Deduplicate while preserving order seen = set() test_dirs = [d for d in test_dirs if not (d in seen or seen.add(d))] + # Native boot + LTP share one VM and one reboot, not independent VMs that + # could run tests against different kernels. The LTP payload runs every + # requested test and reports the boot/baseline result from its evidence. + if test_type_map.get("ltp") in test_dirs: + boot_dirs = {test_type_map.get(kind) for kind in BOOT_TEST_TYPES} + test_dirs = [d for d in test_dirs if d not in boot_dirs or d == test_type_map["ltp"]] + if any(test["type"] == "ltp" for test in job_tests) and not rootfs_url: + raise ValueError("LTP requires artifacts.rootfs containing Kirk and LTP") test_params: Dict[str, str] = { "KERNEL_URL": kernel_url, "MODULES_URL": modules_url, "ARCH": arch, + TESTS_PARAM: json.dumps(job_tests, separators=(",", ":")), } + kernelci = base_config.get("kernelci") or {} + test_params["PULL_LABS_SKIPFILES_JSON"] = json.dumps(kernelci.get("skipfiles") or {}) + revision = jobdef.get("kernel_revision") or {} + test_params["PULL_LABS_CONTEXT_JSON"] = json.dumps( + { + "platform": (jobdef.get("environment") or {}).get("platform", ""), + "branch": revision.get("branch", ""), + "environment": kernelci.get("environment", ""), + } + ) + checksums = (jobdef.get("integrity") or {}).get("sha256") or {} + for name in ("kernel", "modules", "rootfs"): + checksum = checksums.get(name) + if checksum is not None: + if not isinstance(checksum, str) or not re.fullmatch(r"[a-fA-F0-9]{64}", checksum): + raise ValueError(f"Invalid SHA256 for {name}") + test_params[f"{name.upper()}_SHA256"] = checksum.lower() if rootfs_url: test_params["ROOTFS_URL"] = rootfs_url if node_id: @@ -126,15 +234,19 @@ def translate_job( # Pass each individual test's id/type so the vm-test can run them in order. if job_tests: test_params["PULL_LABS_TESTS"] = ",".join( - f"{t.get('id', t.get('type', 'unknown'))}:{t.get('type', 'unknown')}" - for t in job_tests + f"{t.get('id', t.get('type', 'unknown'))}:{t.get('type', 'unknown')}" for t in job_tests ) - timeout = jobdef.get("timeout") or 3600 + # Allow kernel installation/reboot in addition to full suite timeouts when + # the job does not impose an overall deadline. + default_timeout = min(43200, max(3600, sum(test["timeout_s"] for test in job_tests) + 900)) + timeout = jobdef.get("timeout") or default_timeout try: timeout = int(timeout) except (TypeError, ValueError): timeout = 3600 + if not 0 < timeout <= 43200: + raise ValueError("Job timeout must be between 1 and 43200 seconds") # The pipeline groups outputs by test_id; make it unique per job so that # parallel pulls don't collide. @@ -144,7 +256,7 @@ def translate_job( vm_entry: Dict[str, Any] = { "ami_id": plat_defaults["ami_id"], "instance_type": plat_defaults["instance_type"], - "root_volume_size": 40, + "root_volume_size": plat_defaults.get("root_volume_size", 40), "max_runtime": timeout, "test": test_dirs, "min_count": 1, diff --git a/tests/test_aws_auth_setup.py b/tests/test_aws_auth_setup.py index 7330456..140d5ad 100644 --- a/tests/test_aws_auth_setup.py +++ b/tests/test_aws_auth_setup.py @@ -119,3 +119,16 @@ def test_get_credentials(self, mock_session, mock_subprocess, mock_config, mock_ assert "session" in creds assert "region" in creds assert creds["region"] == "us-west-2" + + +def test_runner_image_is_pinned_by_digest(): + auth = object.__new__(AWSAuth) + auth.config = {"ecs": {"task_definition": {"family": "t"}}} + ecr = Mock() + ecr.describe_images.return_value = {"imageDetails": [{"imageDigest": "sha256:" + "d" * 64}]} + uri = auth._use_image_digest( # pylint: disable=protected-access + ecr, "repo", "123.dkr.ecr/repo", "latest" + ) + assert uri == "123.dkr.ecr/repo@sha256:" + "d" * 64 + assert auth.config["ecs"]["task_definition"]["image"] == uri + ecr.describe_images.assert_called_once_with(repositoryName="repo", imageIds=[{"imageTag": "latest"}]) diff --git a/tests/test_native_kernel.py b/tests/test_native_kernel.py new file mode 100644 index 0000000..96f4d6b --- /dev/null +++ b/tests/test_native_kernel.py @@ -0,0 +1,493 @@ +# SPDX-License-Identifier: Apache-2.0 +# pylint: disable=protected-access +"""Native kernel tests must distinguish real boots/tests from scaffold passes.""" + +import gzip +import base64 +import importlib.util +import io +import json +from pathlib import Path +import struct +import sys +import subprocess +import tarfile +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from kernel_ci_cloud_labs.kernel_test_results import ( + parse_runtest_names, + validate_boot_evidence, + validate_kirk_report, + verify_test_evidence, +) +from kernel_ci_cloud_labs.launch_vm import VMLauncher +from kernel_ci_cloud_labs.core.pipeline import parse_evidence_verdicts, parse_vm_logs + +LIB = Path(__file__).parents[1] / "vm-tests/lib" +sys.path.insert(0, str(LIB)) +spec = importlib.util.spec_from_file_location("url_kernel", LIB / "url_kernel.py") +kernel = importlib.util.module_from_spec(spec) +spec.loader.exec_module(kernel) + +RELEASE = "7.3.0-next-test" +VERSION = "#2 SMP PREEMPT_DYNAMIC @1234567" +SHA = "a" * 64 +URL = "https://files.example/kernel" + + +def evidence(): + return { + "status": "pass", + "kernel_url": URL, + "arch": "arm64", + "release": RELEASE, + "requested_release": RELEASE, + "version": VERSION, + "requested_version": VERSION, + "boot_id": "after", + "bootstrap_boot_id": "before", + "image_sha256": SHA, + "download_sha256": "b" * 64, + "cmdline": "root=UUID=root pullab.image_sha256=" + SHA, + } + + +def report(status="pass", kernel_release=RELEASE): + return { + "environment": {"kernel": kernel_release}, + "results": [{"test_fqn": "fork01", "status": status, "test": {"failed": 0, "broken": 0, "warnings": 0}}], + } + + +def write_vm_log(directory, instance_id, test="url-kernel-ltp"): + (directory / f"{instance_id}.log").write_text( + f"test_{test}/input/payload.zip\nTest execution completed: SUCCESS\n" + ) + + +def test_rejected_native_evidence_cannot_become_a_kernelci_pass(tmp_path): + write_vm_log(tmp_path, "i-test", "url-kernel-boot") + result = parse_vm_logs(tmp_path, evidence_tests={"url-kernel-boot"}, verdicts={}) + assert result["successful"] == 0 + assert result["instances"][0]["status"] == "FAIL" + + +def test_native_verdicts_are_judged_per_instance(tmp_path): + write_vm_log(tmp_path, "i-good") + write_vm_log(tmp_path, "i-bad") + write_vm_log(tmp_path, "i-other", "basic-test") + verdicts = { + "i-good": {"result": "PASS", "boot": "PASS", "boot_requested": True}, + "i-bad": {"result": "FAIL", "boot": "PASS", "boot_requested": True}, + } + result = parse_vm_logs(tmp_path, evidence_tests={"url-kernel-ltp"}, verdicts=verdicts) + by_id = {entry["instance_id"]: entry for entry in result["instances"]} + assert by_id["i-good"]["status"] == "PASS" and by_id["i-good"]["boot_status"] == "PASS" + # A failing sibling VM does not reject a verified one, and an LTP failure + # does not take the boot result down with it. + assert by_id["i-bad"]["status"] == "FAIL" and by_id["i-bad"]["boot_status"] == "PASS" + assert by_id["i-other"]["status"] == "PASS" and "boot_status" not in by_id["i-other"] + + +def test_evidence_verdicts_are_read_from_the_container_log(tmp_path): + log = tmp_path / "container.log" + log.write_text( + "[2026-11-03 10:00:00] [CONTAINER] INFO: unrelated\n" + '[2026-11-03 10:00:01] [CONTAINER] PULLAB_EVIDENCE {"instance_id": "i-a", "result": "PASS"}\n' + "[2026-11-03 10:00:02] [CONTAINER] PULLAB_EVIDENCE not-json\n" + ) + assert parse_evidence_verdicts(log) == {"i-a": {"instance_id": "i-a", "result": "PASS"}} + assert parse_evidence_verdicts(tmp_path / "missing.log") == {} + + +def test_kirk_conflicting_results_and_aggregate_failures_are_rejected(): + data = report() + data["stats"] = {"failed": 1} + with pytest.raises(ValueError): + validate_kirk_report(data, RELEASE) + data.pop("stats") + data["results"][0]["test"]["result"] = "brok" + with pytest.raises(ValueError): + validate_kirk_report(data, RELEASE) + + +def make_archive(path, entries): + with tarfile.open(path, "w:gz") as archive: + for name, value in entries.items(): + member = tarfile.TarInfo(name) + if isinstance(value, tuple): + member.type, member.linkname = value + archive.addfile(member) + else: + member.size = len(value) + archive.addfile(member, io.BytesIO(value)) + + +def test_arm64_image_identity_is_read_from_the_supplied_binary(): + image = bytearray(64) + image[56:60] = b"ARMd" + image += ("Linux version " + RELEASE + " (builder@host) (GCC 15) " + VERSION + "\0").encode() + raw, identity = kernel.image_identity(gzip.compress(image)) + assert raw == image + assert identity == {"arch": "arm64", "release": RELEASE, "version": VERSION} + + +def test_x86_image_uses_boot_protocol_version_pointer(): + image = bytearray(0x1000) + image[0x202:0x206] = b"HdrS" + struct.pack_into("", "Norbert Manthey "] __copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved." @@ -6,6 +7,8 @@ from unittest.mock import Mock +import pytest +from botocore.exceptions import ClientError from kernel_ci_cloud_labs.auth.aws_cloudwatch_manager import AWSCloudWatchManager from kernel_ci_cloud_labs.auth.aws_cluster_manager import AWSClusterManager @@ -119,7 +122,10 @@ class TestTaskDefinitionManager: def test_ensure_exists_creates_missing_task_definition(self): """Test that ensure_exists registers task definition when missing""" mock_client = Mock() - mock_client.describe_task_definition.side_effect = Exception("Not found") + mock_client.describe_task_definition.side_effect = ClientError( + {"Error": {"Code": "ClientException", "Message": "Unable to describe task definition."}}, + "DescribeTaskDefinition", + ) mock_client.register_task_definition.return_value = { "taskDefinition": {"taskDefinitionArn": "arn:aws:ecs:us-west-2:123:task-definition/test:1"} } @@ -164,7 +170,10 @@ def test_ensure_exists_skips_existing_task_definition(self): def test_task_definition_includes_cloudwatch_logs(self): """Test that registered task definition includes CloudWatch logs config""" mock_client = Mock() - mock_client.describe_task_definition.side_effect = Exception("Not found") + mock_client.describe_task_definition.side_effect = ClientError( + {"Error": {"Code": "ClientException", "Message": "Unable to describe task definition."}}, + "DescribeTaskDefinition", + ) mock_client.register_task_definition.return_value = { "taskDefinition": {"taskDefinitionArn": "arn:aws:ecs:us-west-2:123:task-definition/test:1"} } @@ -193,3 +202,45 @@ def test_task_definition_includes_cloudwatch_logs(self): assert container_def["logConfiguration"]["logDriver"] == "awslogs" # Log group defaults to /ecs/{family} if not explicitly set assert "/ecs/" in container_def["logConfiguration"]["options"]["awslogs-group"] + + @pytest.mark.parametrize("changed_image", [False, True]) + def test_runner_image_controls_task_revision(self, changed_image): + client = Mock() + manager = AWSTaskDefinitionManager(client, {}) + config = {"image": "runner:new", "execution_role_arn": "exec", "task_role_arn": "task"} + current = manager._definition("test", config) + current.update(status="ACTIVE", taskDefinitionArn="family:1") + if changed_image: + current["containerDefinitions"][0]["image"] = "runner:old" + client.describe_task_definition.return_value = {"taskDefinition": current} + client.register_task_definition.return_value = {"taskDefinition": {"taskDefinitionArn": "family:2"}} + arn, created = manager.ensure_exists("test", config) + assert created is changed_image + assert arn == ("family:2" if changed_image else "family:1") + assert client.register_task_definition.call_count == int(changed_image) + client.deregister_task_definition.assert_not_called() + + def test_aws_normalised_fields_do_not_force_a_new_revision(self): + client = Mock() + manager = AWSTaskDefinitionManager(client, {}) + config = {"image": "runner@sha256:abc", "execution_role_arn": "exec", "task_role_arn": "task"} + current = manager._definition("test", config) + current.update(status="ACTIVE", taskDefinitionArn="family:1", revision=1) + container = current["containerDefinitions"][0] + # Fields describe_task_definition adds to what was registered. + container["logConfiguration"]["secretOptions"] = [] + container["linuxParameters"].update(capabilities={}, devices=[], tmpfs=[]) + container.update(cpu=0, portMappings=[], environment=[], mountPoints=[], volumesFrom=[]) + client.describe_task_definition.return_value = {"taskDefinition": current} + arn, created = manager.ensure_exists("test", config) + assert (arn, created) == ("family:1", False) + client.register_task_definition.assert_not_called() + + def test_access_denied_does_not_register_a_task(self): + client = Mock() + client.describe_task_definition.side_effect = ClientError( + {"Error": {"Code": "AccessDeniedException"}}, "DescribeTaskDefinition" + ) + with pytest.raises(ClientError): + AWSTaskDefinitionManager(client, {}).ensure_exists("test", {"image": "runner:new"}) + client.register_task_definition.assert_not_called() diff --git a/vm-tests/lib/kernel_test_results.py b/vm-tests/lib/kernel_test_results.py new file mode 120000 index 0000000..bcd2260 --- /dev/null +++ b/vm-tests/lib/kernel_test_results.py @@ -0,0 +1 @@ +../../src/kernel_ci_cloud_labs/kernel_test_results.py \ No newline at end of file diff --git a/vm-tests/lib/url_kernel.py b/vm-tests/lib/url_kernel.py new file mode 100644 index 0000000..4a65761 --- /dev/null +++ b/vm-tests/lib/url_kernel.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright (C) 2026 Collabora Limited +"""Install a supplied kernel, verify the reboot, and run requested LTP suites. + +This file is included in each VM payload. Keep it compatible with AL2023's +system Python 3.9; it must work before the project's Python package is installed. +""" + +import argparse +import gzip +import hashlib +import json +import os +import platform +import posixpath +import re +import shutil +import signal +import struct +import subprocess +import sys +import tarfile +import threading +import time +import urllib.request +from pathlib import Path, PurePosixPath + +from kernel_test_results import parse_runtest_names, validate_boot_evidence, validate_kirk_report + + +def digest(path): + result = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + result.update(block) + return result.hexdigest() + + +def write_json(path, data): + path = Path(path) + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + path.chmod(0o644) + + +def download(url, destination, expected_hash=None): + if not isinstance(url, str) or not url.startswith(("https://", "http://")): + raise ValueError("Artifact URL must use HTTP(S)") + destination = Path(destination) + for attempt in range(3): + try: + with urllib.request.urlopen(url, timeout=30) as source, destination.open("wb") as output: + shutil.copyfileobj(source, output, 1024 * 1024) + if destination.stat().st_size == 0: + raise ValueError("Downloaded artifact is empty") + if expected_hash and digest(destination) != expected_hash: + raise ValueError("Artifact SHA256 does not match the job definition") + return + except OSError: + if attempt == 2: + raise + time.sleep(2) + + +def image_identity(data): + """Read the release/build identity embedded in x86 bzImage or arm64 Image.""" + if data.startswith(b"\x1f\x8b"): + data = gzip.decompress(data) + if data[0x202:0x206] == b"HdrS": + if not struct.unpack_from(" tries to change propagation + # before mounting, which fails for a new (not-yet-mounted) guest /proc. + run(["mount", "-t", "proc", "-o", "nosuid,noexec,nodev", "proc", proc]) + for source, guest in [ + ("/dev", "/dev"), + ("/sys", "/sys"), + ("/lib/modules", "/lib/modules"), + (str(output), "/run/pullab-results"), + ]: + target = root_path(root, guest) + target.mkdir(parents=True, exist_ok=True) + run(["mount", "--rbind", source, target]) + run(["mount", "--make-rslave", target]) + runner = request["runner"] + if not runner.startswith("/") or not root_path(root, runner).is_file(): + raise ValueError("Requested Kirk runner is absent from the rootfs") + for suite in request["suites"]: + if not root_path(root, "/opt/ltp/runtest/" + suite).is_file(): + raise ValueError("Requested LTP suite is absent from rootfs: " + suite) + workers = (os.cpu_count() or 1) if request["workers"] == "max" else int(request["workers"]) + command = [ + "chroot", + root, + "/usr/bin/env", + "PATH=/usr/sbin:/usr/bin:/sbin:/bin:/opt/ltp/testcases/bin", + "LTPROOT=/opt/ltp", + "LTP_COLORIZE_OUTPUT=0", + runner, + "--no-colors", + "--run-suite", + ] + request["suites"] + command += [ + "--json-report", + "/run/pullab-results/report.json", + "--workers", + str(min(workers, 128)), + "--suite-timeout", + str(request["timeout_s"]), + ] + if request.get("resolved_skipfile"): + command += ["--skip-file", request["resolved_skipfile"]] + # timeout_s is the request's total budget, enforced by execute_ltp() across + # all suites. No single suite can usefully exceed it, so it also bounds + # Kirk's per-suite timeout. + run(command) + + +def selected_ltp_tests(root, request, skipfile): + patterns = [] + if skipfile: + for line in Path(skipfile).read_text().splitlines(): + if line.strip() and not line.lstrip().startswith("#"): + patterns.append(re.compile(line.strip())) + selected = set() + for suite in request["suites"]: + source = root_path(root, "/opt/ltp/runtest/" + suite) + if not source.is_file(): + raise ValueError("Requested LTP suite is absent from rootfs: " + suite) + for name in parse_runtest_names(source.read_text()): + if not any(pattern.search(name) for pattern in patterns): + selected.add(name) + if not selected: + raise ValueError("Requested suites/skipfile selected no LTP tests") + return sorted(selected) + + +HEARTBEAT_SECONDS = 300 + + +def run_streamed(command, log_path, timeout, label): + """Run *command*, copying its output to *log_path* and to stdout. + + Stdout is the management agent's live console stream, which the AWS side + watches for hangs; long, quiet LTP tests therefore also get a periodic + heartbeat. Returns the exit code, or None after killing the process group + when *timeout* seconds elapse. + """ + with Path(log_path).open("w") as log: + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + start_new_session=True, + text=True, + errors="replace", + ) + + def copy_output(): + for line in process.stdout: + log.write(line) + log.flush() + sys.stdout.write("[" + label + "] " + line) + sys.stdout.flush() + + reader = threading.Thread(target=copy_output, daemon=True) + reader.start() + start = time.monotonic() + deadline = start + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + reader.join(10) + return None + try: + code = process.wait(timeout=min(remaining, HEARTBEAT_SECONDS)) + break + except subprocess.TimeoutExpired: + print("[%s] still running after %ds" % (label, time.monotonic() - start), flush=True) + reader.join(10) + return code + + +def execute_ltp(state_path, state, request, output_dir): + stage = Path(state_path).parent / "kernel-artifacts" + root = stage / "rootfs" + if not root.is_dir(): + download(os.environ["ROOTFS_URL"], stage / "rootfs.tar", os.environ.get("ROOTFS_SHA256")) + safe_extract(stage / "rootfs.tar", root) + output = stage / ("ltp-" + request["id"]) + output.mkdir() + context = json.loads(os.environ.get("PULL_LABS_CONTEXT_JSON", "{}")) + request["resolved_skipfile"] = prepare_skipfile(request, root, output, context) + expected_tests = selected_ltp_tests(root, request, output / "skipfile" if request["resolved_skipfile"] else None) + request_file = output / "request.json" + write_json(request_file, request) + (root / "proc").mkdir(exist_ok=True) + command = [ + "unshare", + "--mount", + "--pid", + "--fork", + "--kill-child", + sys.executable, + str(Path(__file__).resolve()), + "ltp-child", + "--root", + str(root), + "--output", + str(output), + "--request", + str(request_file), + ] + log_path = output_dir / ("results_ltp-" + request["id"] + ".log") + code = run_streamed(command, log_path, request["timeout_s"], "ltp-" + request["id"]) + if code is None: + raise ValueError( + "LTP exceeded its total timeout_s budget of %ds for suites %s" + % (request["timeout_s"], ",".join(request["suites"])) + ) + report_path = output / "report.json" + report = None + if report_path.exists(): + report = json.loads(report_path.read_text()) + report["pullab"] = { + "expected_tests": expected_tests, + "requested_suites": request["suites"], + "image_sha256": state["image_sha256"], + "context": context, + } + write_json(output_dir / ("results_ltp-" + request["id"] + ".json"), report) + if code != 0: + raise ValueError("Kirk execution failed; see " + log_path.name) + if not report_path.is_file(): + raise ValueError("Kirk did not produce a JSON result report") + summary = validate_kirk_report(report, state["release"], expected_tests, expected_version=state["version"]) + print("LTP", request["id"], json.dumps(summary), flush=True) + + +def test(state_path): + state = verify(state_path) + output = Path(state_path).resolve().parent + requests = json.loads(os.environ["PULL_LABS_TESTS_JSON"]) + if not requests: + raise ValueError("No tests were requested") + for request in requests: + if request["type"] in ("boot", "baseline"): + continue # verify() above is the actual baseline test. + if request["type"] != "ltp": + raise ValueError("Unsupported test type: " + request["type"]) + execute_ltp(state_path, state, request, output) + print("All requested tests completed against the verified kernel.", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("install", "test", "ltp-child")) + parser.add_argument("--state") + parser.add_argument("--root") + parser.add_argument("--output") + parser.add_argument("--request") + args = parser.parse_args() + if os.geteuid() != 0: + parser.error("Kernel installation and LTP require root in the disposable test VM") + try: + if args.action == "install": + install(args.state) + elif args.action == "test": + test(args.state) + else: + ltp_child(args.root, args.output, args.request) + except Exception as error: + if args.state: + write_json( + Path(args.state).resolve().parent / "results_error.json", + {"stage": args.action, "status": "fail", "error": str(error)}, + ) + print("ERROR:", error, file=sys.stderr, flush=True) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/vm-tests/test-vm-client.sh b/vm-tests/test-vm-client.sh index a323713..413df9e 100755 --- a/vm-tests/test-vm-client.sh +++ b/vm-tests/test-vm-client.sh @@ -336,6 +336,18 @@ if ! aws s3 cp --no-progress "$SCRIPT_OUTPUT_LOG" "s3://$BUCKET_NAME/${S3_PREFIX fi # Check if script failed (not success and not reboot request) +upload_result_files() +{ + # Keep structured boot/LTP diagnostics on failures as well as successes. + for results_file in "$TEST_DIR"/results_*; do + if [ -f "$results_file" ]; then + results_filename=$(basename "$results_file") + echo "$(date): [RUN_ID:$RUN_ID] Uploading results file: $results_filename" + aws s3 cp --no-progress "$results_file" "s3://$BUCKET_NAME/${S3_PREFIX}/output/$INSTANCE_ID/$results_filename" || true + fi + done +} + if [ $SCRIPT_EXIT_CODE -ne 0 ] && [ $SCRIPT_EXIT_CODE -ne 194 ]; then echo "$(date): [RUN_ID:$RUN_ID] ERROR: Script failed - stopping execution chain" @@ -364,6 +376,7 @@ EOF # Upload failure status files aws s3 cp --no-progress "$LOG_DIR/result.txt" "s3://$BUCKET_NAME/${S3_PREFIX}/output/$INSTANCE_ID/result.txt" || true aws s3 cp --no-progress "$LOG_DIR/stats.json" "s3://$BUCKET_NAME/${S3_PREFIX}/output/$INSTANCE_ID/stats.json" || true + upload_result_files echo "$(date): [RUN_ID:$RUN_ID] Test execution failed: $RESULT" @@ -421,16 +434,7 @@ EOF fi done - # Upload all results_* files from the test directory - for results_file in "$TEST_DIR"/results_*; do - if [ -f "$results_file" ]; then - results_filename=$(basename "$results_file") - echo "$(date): [RUN_ID:$RUN_ID] Uploading results file: $results_filename" - if ! aws s3 cp --no-progress "$results_file" "s3://$BUCKET_NAME/${S3_PREFIX}/output/$INSTANCE_ID/$results_filename"; then - echo "$(date): [RUN_ID:$RUN_ID] WARNING: Failed to upload $results_filename" - fi - fi - done + upload_result_files echo "$(date): [RUN_ID:$RUN_ID] Test execution completed: $RESULT" diff --git a/vm-tests/url-kernel-boot/README.md b/vm-tests/url-kernel-boot/README.md index 4e45bb4..77d8551 100644 --- a/vm-tests/url-kernel-boot/README.md +++ b/vm-tests/url-kernel-boot/README.md @@ -1,31 +1,28 @@ -# url-kernel-boot +# Native KernelCI boot test -A vm-test that downloads a kernel image, modules tarball, and optional -rootfs from URLs provided in environment variables, then runs a -benchmark against the resulting environment. +This test installs the supplied kernel and its modules on a disposable Amazon +Linux 2023 VM. Stage 1 generates an initramfs with dracut, adds a grubby entry +and schedules it for exactly one boot (`grub2-reboot`), and lets the VM client +reboot through SSM exit code 194. A kernel that panics therefore falls back to +the AMI kernel and fails verification instead of looping. Stage 2 +requires a new boot ID, the release and build version embedded in the supplied +image, and its SHA256 marker on the kernel command line. AMI fallback fails. -Used by `pull_labs_translate.translate_job` to consume PULL_LABS -`artifacts.kernel` / `artifacts.modules` / `artifacts.rootfs` URLs from a -KernelCI job definition. +Supported images are x86_64 bzImage and arm64 Image or Image.gz. Modules must +be a tar archive containing exactly the matching lib/modules release tree. +The AMI userspace and SSM agent remain available for management; the supplied +kernel must support the VM's root filesystem, storage, networking, EFI, and +initramfs. Missing drivers or an incompatible image are test failures. -## Environment variables +Required environment: `KERNEL_URL`, `MODULES_URL`, `ARCH`, and +`PULL_LABS_TESTS_JSON`. Optional `KERNEL_SHA256` and `MODULES_SHA256` validate +artifact downloads. The translator constructs these values. -| Variable | Required | Purpose | -| --- | --- | --- | -| `KERNEL_URL` | yes | URL to the kernel image (e.g. bzImage / vmlinuz) | -| `MODULES_URL` | yes | URL to the modules tarball (tar.gz) | -| `ROOTFS_URL` | no | Optional URL to a rootfs / ramdisk image | -| `ARCH` | no | Target architecture string (default: `uname -m`) | -| `KERNELCI_NODE_ID` | no | Job node ID from kernelci-api (recorded in logs) | -| `PULL_LABS_TESTS` | no | Comma-separated `id:type` list from the job definition | -| `BOOT_HOOK` | no | Path to an executable that stages the kernel and reboots/kexecs | +`results_boot.json` records the requested artifact, image hash, release/build, +boot IDs and command line. `results_error.json` records installation or +verification failures. The VM client uploads diagnostic results on failure as +well as success. The AWS runner independently validates the boot evidence; +`SUCCESS` without evidence cannot pass. No synthetic benchmark CSV is emitted. -## Output - -A CSV at `benchmark-tip-.csv` matching the -`metric,unit,value,more_is_better,kernel_version,instance_id,instance_type,arch` -schema consumed by `BenchmarkAnalyzer`. - -The scaffold emits a single `pullab.boot_check` row; replace or extend -the trailing CSV-write block with actual benchmark output (UnixBench, -LTP, etc.) to produce real results. +Use `url-kernel-ltp` for jobs requesting LTP. These stages require root and +modify the bootloader: run them only in disposable test VMs. diff --git a/vm-tests/url-kernel-boot/kernel_test_results.py b/vm-tests/url-kernel-boot/kernel_test_results.py new file mode 120000 index 0000000..3b94c0b --- /dev/null +++ b/vm-tests/url-kernel-boot/kernel_test_results.py @@ -0,0 +1 @@ +../lib/kernel_test_results.py \ No newline at end of file diff --git a/vm-tests/url-kernel-boot/run-01-install-kernel.sh b/vm-tests/url-kernel-boot/run-01-install-kernel.sh new file mode 100644 index 0000000..ad2fa8c --- /dev/null +++ b/vm-tests/url-kernel-boot/run-01-install-kernel.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# These tools operate on the supplied kernel and modules, not the AMI kernel. +sudo dnf install -y dracut grubby kmod python3 python3-pyyaml util-linux +sudo -E python3 "$SOURCE_DIR/url_kernel.py" install --state "$SOURCE_DIR/kernel-state.json" +# The VM client returns SSM's reboot code 194 between these two run scripts. diff --git a/vm-tests/url-kernel-boot/run-02-verify-and-test.sh b/vm-tests/url-kernel-boot/run-02-verify-and-test.sh new file mode 100644 index 0000000..5701b1f --- /dev/null +++ b/vm-tests/url-kernel-boot/run-02-verify-and-test.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +sudo -E python3 "$SOURCE_DIR/url_kernel.py" test --state "$SOURCE_DIR/kernel-state.json" diff --git a/vm-tests/url-kernel-boot/run.sh b/vm-tests/url-kernel-boot/run.sh deleted file mode 100755 index 374ddb9..0000000 --- a/vm-tests/url-kernel-boot/run.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2026 Collabora Limited -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Author: Denys Fedoryshchenko -# Co-Author: Max Hubmann -# Co-Author: Norbert Manthey -# -# Co-author credit: shell scaffolding (set -euxo pipefail, SOURCE_DIR idiom) -# borrowed from vm-tests/basic-test/run.sh and simple-unixbench/common_lib.sh. -# -# vm-test invoked by pull_labs_poller. Reads kernel/modules URLs from -# environment variables (populated from PULL_LABS job_definition by -# pull_labs_translate.translate_job) and stages them on the VM. The actual -# kernel boot + benchmark step is environment-specific and left as a -# parameterised hook; this scaffold validates the download and emits a -# CSV in the same format BenchmarkAnalyzer expects. - -set -euxo pipefail - -: "${KERNEL_URL:?KERNEL_URL is required (set by pull_labs_translate)}" -: "${MODULES_URL:?MODULES_URL is required (set by pull_labs_translate)}" - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -STAGE_DIR="${STAGE_DIR:-/tmp/pullab_kernel}" -mkdir -p "$STAGE_DIR" - -echo "=== url-kernel-boot ===" -echo "Kernel URL: $KERNEL_URL" -echo "Modules URL: $MODULES_URL" -echo "Rootfs URL: ${ROOTFS_URL:-}" -echo "Arch: ${ARCH:-$(uname -m)}" -echo "Node ID: ${KERNELCI_NODE_ID:-unknown}" -echo "Test list: ${PULL_LABS_TESTS:-}" - -download() { - local url="$1" - local dest="$2" - echo "Downloading $url -> $dest" - if command -v curl >/dev/null 2>&1; then - curl -fSL --retry 3 --retry-delay 5 -o "$dest" "$url" - elif command -v wget >/dev/null 2>&1; then - wget -q -O "$dest" "$url" - else - echo "ERROR: neither curl nor wget available" >&2 - exit 2 - fi -} - -KERNEL_BIN="$STAGE_DIR/kernel.bin" -MODULES_TGZ="$STAGE_DIR/modules.tar.gz" - -download "$KERNEL_URL" "$KERNEL_BIN" -download "$MODULES_URL" "$MODULES_TGZ" - -if [ -n "${ROOTFS_URL:-}" ]; then - ROOTFS_BIN="$STAGE_DIR/rootfs.bin" - download "$ROOTFS_URL" "$ROOTFS_BIN" -fi - -echo "Sizes:" -ls -la "$STAGE_DIR" -echo "Kernel file type:" -file "$KERNEL_BIN" || true - -# Hook for the deployment-specific boot step. In an environment that can -# kexec or reboot into the supplied kernel, set BOOT_HOOK to a script that -# stages the kernel into /boot and either kexecs or schedules a reboot. -if [ -n "${BOOT_HOOK:-}" ] && [ -x "${BOOT_HOOK}" ]; then - echo "Invoking BOOT_HOOK: $BOOT_HOOK" - "$BOOT_HOOK" "$KERNEL_BIN" "$MODULES_TGZ" "${ROOTFS_BIN:-}" -else - echo "No BOOT_HOOK configured — running tests against currently booted kernel." -fi - -# Emit a minimal benchmark CSV so BenchmarkAnalyzer sees at least one row. -# Real benchmark scripts should append to this file using the same schema. -RESULT_CSV="${SOURCE_DIR}/benchmark-tip-$(uname -r).csv" -echo "metric,unit,value,more_is_better,kernel_version,instance_id,instance_type,arch" > "$RESULT_CSV" -echo "pullab.boot_check,bool,1,true,$(uname -r),${HOSTNAME:-unknown},${INSTANCE_TYPE:-unknown},${ARCH:-$(uname -m)}" >> "$RESULT_CSV" - -echo "Test execution completed: SUCCESS" diff --git a/vm-tests/url-kernel-boot/url_kernel.py b/vm-tests/url-kernel-boot/url_kernel.py new file mode 120000 index 0000000..ef4f64c --- /dev/null +++ b/vm-tests/url-kernel-boot/url_kernel.py @@ -0,0 +1 @@ +../lib/url_kernel.py \ No newline at end of file diff --git a/vm-tests/url-kernel-ltp/README.md b/vm-tests/url-kernel-ltp/README.md new file mode 100644 index 0000000..9ab74b5 --- /dev/null +++ b/vm-tests/url-kernel-ltp/README.md @@ -0,0 +1,42 @@ +# LTP against the supplied kernel + +This payload shares the two native boot stages with `url-kernel-boot`. After +verifying the requested kernel, it extracts the job's `ROOTFS_URL` tar archive +and runs the rootfs's Kirk and LTP in a chroot with private mount/PID namespaces. +The rootfs must contain `/opt/kirk/kirk` and `/opt/ltp/runtest/`. + +The translator preserves test dependencies, suite names (`tst_cmdfiles`, comma +separated), workers (1–128 or `max`), skipfile and timeout. Unsupported test +types, parameters and pre/post commands fail explicitly. Each requested LTP +test produces a complete Kirk JSON report and a log. Every selected test must +appear in the report; failures, broken tests, warnings, empty reports, and +all-skipped runs fail even if Kirk exits zero. `conf`/`skip` are accepted when +at least one selected test passes. The report must identify the booted kernel. + +A test's `timeout_s` is its total budget across all requested suites; it also +bounds Kirk's per-suite timeout. Kirk output is mirrored to the VM console +stream, with a heartbeat every five minutes, so long quiet suites are not +mistaken for hangs. When a job requests `baseline`/`boot` as well as LTP, one +VM runs both and reports a separate `boot` result from the boot evidence. + +Named skipfiles must exist in the rootfs or have an explicit deployment URL. +For the current KernelCI rootfs, configure the pinned LKFT skipfile, for example: + +```json +{ + "kernelci": { + "environment": "staging", + "skipfiles": { + "skipfile-lkft.yaml": "https://raw.githubusercontent.com/Linaro/test-definitions/a6247b6442319af519b8760421a50cb504351eec/automated/linux/ltp/skipfile-lkft.yaml" + } + } +} +``` + +LKFT YAML selectors use the job's platform, kernel branch and configured +environment. Test names become anchored regexes for Kirk. Missing skipfiles +fail rather than silently changing the test selection. An explicit empty name +or `/dev/null` selects no skips. The final reports include the selected tests, +suites, context and image hash for independent AWS-side validation. + +Run only on disposable VMs. LTP executes as root and exercises the real kernel. diff --git a/vm-tests/url-kernel-ltp/external_requirements.json b/vm-tests/url-kernel-ltp/external_requirements.json new file mode 100644 index 0000000..87fa49d --- /dev/null +++ b/vm-tests/url-kernel-ltp/external_requirements.json @@ -0,0 +1 @@ +{"kernel-rpms/src": false, "kernel-rpms/binary": false} diff --git a/vm-tests/url-kernel-ltp/kernel_test_results.py b/vm-tests/url-kernel-ltp/kernel_test_results.py new file mode 120000 index 0000000..3b94c0b --- /dev/null +++ b/vm-tests/url-kernel-ltp/kernel_test_results.py @@ -0,0 +1 @@ +../lib/kernel_test_results.py \ No newline at end of file diff --git a/vm-tests/url-kernel-ltp/run-01-install-kernel.sh b/vm-tests/url-kernel-ltp/run-01-install-kernel.sh new file mode 120000 index 0000000..7141663 --- /dev/null +++ b/vm-tests/url-kernel-ltp/run-01-install-kernel.sh @@ -0,0 +1 @@ +../url-kernel-boot/run-01-install-kernel.sh \ No newline at end of file diff --git a/vm-tests/url-kernel-ltp/run-02-verify-and-test.sh b/vm-tests/url-kernel-ltp/run-02-verify-and-test.sh new file mode 120000 index 0000000..37b6844 --- /dev/null +++ b/vm-tests/url-kernel-ltp/run-02-verify-and-test.sh @@ -0,0 +1 @@ +../url-kernel-boot/run-02-verify-and-test.sh \ No newline at end of file diff --git a/vm-tests/url-kernel-ltp/url_kernel.py b/vm-tests/url-kernel-ltp/url_kernel.py new file mode 120000 index 0000000..ef4f64c --- /dev/null +++ b/vm-tests/url-kernel-ltp/url_kernel.py @@ -0,0 +1 @@ +../lib/url_kernel.py \ No newline at end of file