Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ MONITOR_STORAGE_CLASS ?=
INSTALL_MONITOR ?=
INSTALL_DRAGONFLY ?=
ENABLE_RUNC ?=
SCHEDULE_PLACEMENT_POLICY ?=
GRAFANA_PUBLIC_ACCESS ?=
GRAFANA_ADMIN_PASSWORD ?=
IAM_SEED_HEX ?=
Expand All @@ -49,6 +50,7 @@ help:
@echo " make config NON_INTERACTIVE=1 ... Generate config from Make variables"
@echo " make config INSTALL_DRAGONFLY=true Enable optional P2P image distribution"
@echo " make config ENABLE_RUNC=true Build and register the optional runc runtime"
@echo " make config SCHEDULE_PLACEMENT_POLICY=binpack Use compact scheduling"
@echo " make build IMAGE_TAG=<tag> Build the all-in-one image"
@echo " make build RUNTIME_PROFILE=python Include optional Python runtimes"
@echo " make build AKERNEL_ENABLE_KATA=false Exclude the optional Kata payload"
Expand Down Expand Up @@ -92,6 +94,7 @@ config:
if [[ -n "$(INSTALL_MONITOR)" ]]; then args+=(--install-monitor "$(INSTALL_MONITOR)"); fi; \
if [[ -n "$(INSTALL_DRAGONFLY)" ]]; then args+=(--install-dragonfly "$(INSTALL_DRAGONFLY)"); fi; \
if [[ -n "$(ENABLE_RUNC)" ]]; then args+=(--enable-runc "$(ENABLE_RUNC)"); fi; \
if [[ -n "$(SCHEDULE_PLACEMENT_POLICY)" ]]; then args+=(--schedule-placement-policy "$(SCHEDULE_PLACEMENT_POLICY)"); fi; \
if [[ -n "$(GRAFANA_PUBLIC_ACCESS)" ]]; then args+=(--grafana-public-access "$(GRAFANA_PUBLIC_ACCESS)"); fi; \
if [[ -n "$(GRAFANA_ADMIN_PASSWORD)" ]]; then args+=(--grafana-admin-password "$(GRAFANA_ADMIN_PASSWORD)"); fi; \
if [[ -n "$(IAM_SEED_HEX)" ]]; then args+=(--iam-seed-hex "$(IAM_SEED_HEX)"); fi; \
Expand Down
6 changes: 3 additions & 3 deletions builder/node.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ ARG AKERNEL_ENABLE_KATA=true
ARG AKERNEL_ENABLE_RUNC=false
ARG AKERNEL_ENABLE_FIRECRACKER=true
ARG SANDBOXD_BUILD_IMAGE=golang:1.25.5-bookworm
ARG OPEN_YR_VERSION=0.10.2rc6
ARG OPEN_YR_VERSION=0.10.2rc9
ARG OPEN_YR_CORE_WHEEL_URL=
ARG OPEN_YR_CORE_WHEEL_SHA256=
ARG OPEN_YR_RELEASE_BASE_URL=https://github.com/openYuanrong-mirror/yuanrong/releases/download
ARG OPEN_YR_CORE_AMD64_SHA256=4dda061daf1628b03af559dd114571998490b9dad826cb3e2dee9326a86aa4f7
ARG OPEN_YR_CORE_ARM64_SHA256=bb83a66368168111bbd88af71f9fe009c9aef0ea0b87b38d3f47277f0f678f58
ARG OPEN_YR_CORE_AMD64_SHA256=ced5668500ca8fa7c100690fac5e0877a90b26b7ce1347c81b1b4bc1174abf16
ARG OPEN_YR_CORE_ARM64_SHA256=084e0bfc115985910ac6c9141126ad856f6b16b87843089441ea352927a56ec4
ARG GVISOR_DOWNLOAD_IMAGE=ubuntu:24.04
ARG GVISOR_RELEASE
ARG GVISOR_AMD64_URL
Expand Down
2 changes: 1 addition & 1 deletion builder/runtime.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ ARG PYTHON_311_VERSION=3.11.13
ARG PYTHON_312_VERSION=3.12.11
ARG PYTHON_313_VERSION=3.13.5
ARG PYTHON_314_VERSION=3.14.6
ARG OPEN_YR_VERSION=0.10.2rc6
ARG OPEN_YR_VERSION=0.10.2rc9
ARG OPEN_YR_LEGACY_SDK_VERSION=0.9.9

FROM ${AKERNEL_RUNTIME_BASE_IMAGE} AS rrt-download
Expand Down
10 changes: 10 additions & 0 deletions builder/scripts/master_entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,22 @@ case "${AKERNEL_ROLE:-master}" in
;;
esac
ADVERTISE_FRONTEND_PROXY_CREATE="${ADVERTISE_FRONTEND_PROXY_CREATE:-${advertise_frontend_proxy_create_default}}"
SCHEDULE_PLACEMENT_POLICY="${SCHEDULE_PLACEMENT_POLICY:-spread}"
case "${SCHEDULE_PLACEMENT_POLICY}" in
binpack|spread)
;;
*)
echo "SCHEDULE_PLACEMENT_POLICY must be binpack or spread" >&2
exit 1
;;
esac

exec "${YR_BIN}" start --master --block true \
-e -c 0 -m 8000 -s 4096 -n $HOSTNAME \
-d $DEPLOY_PATH \
--fs_health_check_retry_interval 1 \
--schedule_relaxed 20 \
--schedule_placement_policy "${SCHEDULE_PLACEMENT_POLICY}" \
--enable_faas_frontend ${ENABLE_FAAS_FRONTEND:-true} \
--enable_function_scheduler ${ENABLE_FUNCTION_SCHEDULER:-false} \
--enable_meta_service ${ENABLE_META_SERVICE:-true} \
Expand Down
15 changes: 14 additions & 1 deletion builder/scripts/yr_node_bootstrap.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,19 @@ ulimit -n 32768
export YR_RUNTIME_BACKEND=sandboxd
export YR_IMAGE_PROCESS_CONFIG="${YR_IMAGE_PROCESS_CONFIG:-/run/akernel/yr-image-process.json}"

case "${SCHEDULE_PLACEMENT_POLICY:-spread}" in
spread)
ADVERTISE_FRONTEND_PROXY_CREATE=true
;;
binpack)
ADVERTISE_FRONTEND_PROXY_CREATE=false
;;
*)
echo "SCHEDULE_PLACEMENT_POLICY must be binpack or spread" >&2
exit 1
;;
esac

resolve_node_ip() {
local default_device
local node_ip
Expand Down Expand Up @@ -150,7 +163,7 @@ else
--log_expiration_max_file_count 50 \
--function_proxy_merge_process_enable true \
--enable_direct_routing false \
--advertise_frontend_proxy_create false \
--advertise_frontend_proxy_create "${ADVERTISE_FRONTEND_PROXY_CREATE}" \
Comment thread
WenYuLuo marked this conversation as resolved.
--force_low_reliability_instance true \
--snapshot_storage_mode local_only \
--checkpoint_dir "${CHECKPOINT_DIR}" \
Expand Down
2 changes: 1 addition & 1 deletion builder/systemd_services/yuanrong.service
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Description=yuanrong.service
[Service]
#Type=simple
PIDFile=/run/yuanrong.pid
PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT
PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT SCHEDULE_PLACEMENT_POLICY
Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock"
Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime"
Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1"
Expand Down
27 changes: 27 additions & 0 deletions deploy/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,33 @@ image:
Each component can still override `master.image`, `frontend.image`, or
`node.image` when a split-image deployment is required.

### Sandbox placement policy

YuanRong's native default is `binpack`, but AKernel explicitly defaults to
`spread` so new sandboxes are distributed across eligible nodes. To compact
placements onto fewer nodes instead, set the core chart value:

```yaml
core:
master:
schedulePlacementPolicy: binpack
```

When installing the core chart directly, omit the `core` wrapper. Terraform
deployments use `schedule_placement_policy = "binpack"`. Guided profiles accept
the same choice with:

```bash
make config SCHEDULE_PLACEMENT_POLICY=binpack
```

Only `binpack` and `spread` are accepted. Changing the policy requires the
master and node workloads to restart. The policy is passed to the master
scheduler; node proxies advertise frontend create only for `spread`, while
`binpack` keeps creation on the frontend. It affects subsequent placements and
does not move running sandboxes. Standalone always enables local frontend
create and does not expose a cluster placement setting.

### Public Traefik entrypoints

For cloud deployments, use Traefik with two public entrypoints:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
{{- $schedulePlacementPolicy := .Values.master.schedulePlacementPolicy | default "spread" -}}
{{- if not (has $schedulePlacementPolicy (list "binpack" "spread")) -}}
{{- fail "master.schedulePlacementPolicy must be binpack or spread" -}}
{{- end -}}
{{- if .Values.kruise.enabled }}
apiVersion: apps.kruise.io/v1alpha1
{{- else }}
Expand Down Expand Up @@ -79,6 +83,8 @@ spec:
env:
- name: AKERNEL_ROLE
value: "master"
- name: SCHEDULE_PLACEMENT_POLICY
value: {{ $schedulePlacementPolicy | quote }}
{{- if .Values.frontend.enabled }}
- name: ENABLE_FAAS_FRONTEND
value: "false"
Expand Down
6 changes: 6 additions & 0 deletions deploy/akernel/charts/core/templates/node/daemonset.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
{{- $schedulePlacementPolicy := .Values.master.schedulePlacementPolicy | default "spread" -}}
{{- if not (has $schedulePlacementPolicy (list "binpack" "spread")) -}}
{{- fail "master.schedulePlacementPolicy must be binpack or spread" -}}
{{- end -}}
{{- if .Values.kruise.enabled }}
apiVersion: apps.kruise.io/v1alpha1
{{- else }}
Expand Down Expand Up @@ -87,6 +91,8 @@ spec:
value: "/run/akernel"
- name: YR_IMAGE_PROCESS_CONFIG
value: {{ .Values.yuanrong.imageProcessConfig | default "/run/akernel/yr-image-process.json" | quote }}
- name: SCHEDULE_PLACEMENT_POLICY
value: {{ $schedulePlacementPolicy | quote }}
- name: ETCD_ADDRESS
value: {{ get $nodeEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) | quote }}
- name: ETCD_PORT
Expand Down
3 changes: 3 additions & 0 deletions deploy/akernel/charts/core/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ etcd:

master:
replicas: 1
# AKernel defaults YuanRong's Unit scheduler to spread. Use binpack to
# compact new sandbox placements onto fewer eligible nodes.
schedulePlacementPolicy: spread
host: ""
port: "22770"
service:
Expand Down
25 changes: 25 additions & 0 deletions deploy/scripts/configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ image_tag_override=""
install_monitor_override=""
install_dragonfly_override=""
enable_runc_override=""
schedule_placement_policy_override=""
grafana_public_access_override=""
grafana_admin_password_override=""
iam_seed_hex_override=""
Expand Down Expand Up @@ -121,6 +122,10 @@ while [[ $# -gt 0 ]]; do
enable_runc_override="$2"
shift 2
;;
--schedule-placement-policy)
schedule_placement_policy_override="$2"
shift 2
;;
--grafana-public-access)
grafana_public_access_override="$2"
shift 2
Expand Down Expand Up @@ -183,6 +188,19 @@ normalize_bool() {
esac
}

normalize_schedule_placement_policy() {
local policy
policy="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')"
case "${policy}" in
binpack | spread)
printf '%s' "${policy}"
;;
*)
die "invalid schedule placement policy: $1 (expected binpack or spread)"
;;
esac
}

default_tag="$(git -C "${AKERNEL_REPO_ROOT}" rev-parse --short HEAD)-$(date +%Y%m%d%H%M%S)"
default_cluster_name="akernel"
if [[ "${env_name}" != "default" ]]; then
Expand Down Expand Up @@ -221,6 +239,9 @@ set_or_prompt image_tag "All-in-one image tag" "${default_tag}" "${image_tag_ove
set_or_prompt install_monitor "Install monitor chart (true/false)" "true" "${install_monitor_override}"
set_or_prompt install_dragonfly "Install Dragonfly and dedicated node pools (true/false)" "false" "${install_dragonfly_override}"
set_or_prompt enable_runc "Enable the optional runc runtime (true/false)" "false" "${enable_runc_override}"
set_or_prompt schedule_placement_policy \
"YuanRong schedule placement policy (binpack/spread)" "spread" \
"${schedule_placement_policy_override}"
set_or_prompt grafana_public_access "Expose Grafana LoadBalancer (true/false)" "true" "${grafana_public_access_override}"
set_or_prompt grafana_admin_password \
"Grafana admin password (empty to generate)" "" \
Expand All @@ -231,6 +252,7 @@ fi
install_monitor="$(normalize_bool "${install_monitor}")"
install_dragonfly="$(normalize_bool "${install_dragonfly}")"
enable_runc="$(normalize_bool "${enable_runc}")"
schedule_placement_policy="$(normalize_schedule_placement_policy "${schedule_placement_policy}")"
grafana_public_access="$(normalize_bool "${grafana_public_access}")"

dir="$(state_dir "${env_name}")"
Expand Down Expand Up @@ -352,6 +374,7 @@ grafana_admin_password = "${grafana_admin_password}"

install_dragonfly = ${install_dragonfly}
enable_runc = ${enable_runc}
schedule_placement_policy = "${schedule_placement_policy}"
EOF
;;
huaweicloud)
Expand Down Expand Up @@ -406,6 +429,7 @@ grafana_admin_password = "${grafana_admin_password}"

install_dragonfly = ${install_dragonfly}
enable_runc = ${enable_runc}
schedule_placement_policy = "${schedule_placement_policy}"
EOF
;;
esac
Expand All @@ -427,6 +451,7 @@ CORE_NAMESPACE=akernel
MONITOR_NAMESPACE=akernel-monitor
INSTALL_DRAGONFLY=${install_dragonfly}
AKERNEL_ENABLE_RUNC=${enable_runc}
SCHEDULE_PLACEMENT_POLICY=${schedule_placement_policy}
EOF

chmod 600 "${tfvars_file}" "${config_file}"
Expand Down
1 change: 1 addition & 0 deletions deploy/terraform/aliyun/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ locals {
etcd_image_tag = var.etcd_image_tag
master_image_repository = local.master_image_repo
master_image_tag = var.master_image_tag
schedule_placement_policy = var.schedule_placement_policy
node_image_repository = local.node_image_repo
node_image_tag = var.node_image_tag
traefik_image_repository = local.traefik_image_repo
Expand Down
3 changes: 3 additions & 0 deletions deploy/terraform/aliyun/terraform.tfvars.example
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,9 @@ node_secret_create = true
# matching AKERNEL_ENABLE_RUNC=true profile setting.
enable_runc = false

# AKernel defaults YuanRong placement to "spread"; use "binpack" to compact.
schedule_placement_policy = "spread"

# --- Monitor ---
install_monitor = true
monitor_namespace = "akernel-monitor"
Expand Down
1 change: 1 addition & 0 deletions deploy/terraform/aliyun/values-akernel.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ etcd:

master:
replicas: ${master_replicas}
schedulePlacementPolicy: "${schedule_placement_policy}"
image:
repository: "${master_image_repository}"
tag: "${master_image_tag}"
Expand Down
11 changes: 11 additions & 0 deletions deploy/terraform/aliyun/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,17 @@ variable "sandboxd_nat_backend" {
default = "iptables"
}

variable "schedule_placement_policy" {
type = string
description = "YuanRong Unit scheduler placement policy. AKernel defaults to spread; use binpack for compact placement."
default = "spread"

validation {
condition = contains(["binpack", "spread"], var.schedule_placement_policy)
error_message = "schedule_placement_policy must be binpack or spread."
}
}

variable "enable_runc" {
type = bool
description = "Request the optional runc runtime; the selected node image must be built with AKERNEL_ENABLE_RUNC=true."
Expand Down
1 change: 1 addition & 0 deletions deploy/terraform/huaweicloud/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ locals {
etcd_image_tag = var.etcd_image_tag
master_image_repository = var.master_image_repository
master_image_tag = var.master_image_tag
schedule_placement_policy = var.schedule_placement_policy
node_image_repository = var.node_image_repository
node_image_tag = var.node_image_tag
traefik_image_repository = var.traefik_image_repository
Expand Down
3 changes: 3 additions & 0 deletions deploy/terraform/huaweicloud/terraform.tfvars.example
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ node_image_tag = "<release-tag>"
# matching AKERNEL_ENABLE_RUNC=true profile setting.
enable_runc = false

# AKernel defaults YuanRong placement to "spread"; use "binpack" to compact.
schedule_placement_policy = "spread"

# Enable the /internal-stats endpoint sidecar on Traefik.
# traefik_internal_stats_enabled = true
# traefik_internal_stats_image = "my-registry.example.com/busybox:1.37.0-musl"
Expand Down
1 change: 1 addition & 0 deletions deploy/terraform/huaweicloud/values-akernel.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ etcd:

master:
replicas: ${master_replicas}
schedulePlacementPolicy: "${schedule_placement_policy}"
image:
repository: "${master_image_repository}"
tag: "${master_image_tag}"
Expand Down
11 changes: 11 additions & 0 deletions deploy/terraform/huaweicloud/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,17 @@ variable "sandboxd_nat_backend" {
default = "iptables"
}

variable "schedule_placement_policy" {
type = string
description = "YuanRong Unit scheduler placement policy. AKernel defaults to spread; use binpack for compact placement."
default = "spread"

validation {
condition = contains(["binpack", "spread"], var.schedule_placement_policy)
error_message = "schedule_placement_policy must be binpack or spread."
}
}

variable "enable_runc" {
type = bool
description = "Request the optional runc runtime; the selected node image must be built with AKERNEL_ENABLE_RUNC=true."
Expand Down
4 changes: 3 additions & 1 deletion sdk/python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -646,7 +646,9 @@ for node in resources():

Accelerators appear under keys such as `GPU/l20`. Capacity is the total card
count and allocatable is the currently free count. `ak resources` renders the
same information as, for example, `gpu/l20 1/4`.
same information as, for example, `gpu/l20 1/4`. Its status column renders only
normal nodes as `OK`; unavailable states remain explicit as `EVICTING`,
`RECOVERING`, or `TO_BE_DELETED`.

Use a context manager or call `kill()` in a `finally` block. The SDK does not
delete sandboxes during garbage collection or interpreter exit. Cleanup failures
Expand Down
Loading
Loading