Let ArchiveBox 0.9 actually crawl, and serve its snapshots

Every extractor binary 0.9 cannot find aborts the crawl runner, and abx-plugins
keeps adding new ones, so a fixed apt list always falls behind while the service
user cannot run apt. archivebox install now runs as root with the service user's
home, the way upstream's own package does, and resolves every system dependency
itself; mercury stays disabled because npm refuses its git dependency. Snapshot
iframes are built from BASE_URL, so the update sets it when missing or when it is
a stale IP, and migrated 0.7 collections get their snapshots re-indexed.

ct/archivebox.sh

@@ -13,7 +13,7 @@ var_cpu="${var_cpu:-2}"
 var_ram="${var_ram:-1024}"
 var_disk="${var_disk:-8}"
 var_os="${var_os:-debian}"
-var_version="${var_version:-12}"
+var_version="${var_version:-13}"
 var_arm64="${var_arm64:-yes}"
 var_unprivileged="${var_unprivileged:-1}"
 
@@ -31,26 +31,75 @@ function update_script() {
     exit
   fi
 
-  NODE_VERSION="22" NODE_MODULE="@postlight/parser@latest,single-file-cli@latest" setup_nodejs
-  PYTHON_VERSION="3.13" setup_uv
-
-  ensure_dependencies chromium
+  setup_uv
 
   msg_info "Stopping Service"
   systemctl stop archivebox
   msg_ok "Stopped Service"
 
-  msg_info "Upgrading Playwright"
-  $STD uv pip install playwright --system --break-system-packages
-  $STD playwright install-deps chromium
-  msg_ok "Upgraded Playwright"
+  local migrate=0 py port base runtime
+  if [[ ! -x /opt/archivebox/venv/bin/archivebox ]]; then
+    migrate=1
+    msg_info "Moving ArchiveBox to its own Python 3.13 environment"
+    py="/usr/bin/python3.13"
+    [[ -x "$py" ]] || ensure_dependencies python3.13 || true
+    if [[ ! -x "$py" ]]; then
+      $STD uv python install --install-dir /opt/archivebox/python 3.13
+      py="$(UV_PYTHON_INSTALL_DIR=/opt/archivebox/python uv python find 3.13)"
+    fi
+    $STD uv venv --python "$py" /opt/archivebox/venv
+    msg_ok "Moved ArchiveBox to its own Python 3.13 environment"
+  fi
+
+  port="$(awk -F: '/^ExecStart=/ {print $NF}' /etc/systemd/system/archivebox.service 2>/dev/null)"
+  [[ "$port" =~ ^[0-9]+$ ]] || port=5797
+  cat <<EOF >/etc/systemd/system/archivebox.service
+[Unit]
+Description=ArchiveBox Server
+After=network.target
+
+[Service]
+User=archivebox
+WorkingDirectory=/opt/archivebox/data
+ExecStartPre=/opt/archivebox/venv/bin/archivebox init
+ExecStart=/opt/archivebox/venv/bin/archivebox server 0.0.0.0:${port}
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+EOF
+  systemctl daemon-reload
 
   msg_info "Updating ArchiveBox"
+  $STD uv pip install --python /opt/archivebox/venv/bin/python --upgrade archivebox
+  ln -sf /opt/archivebox/venv/bin/archivebox /usr/local/bin/archivebox
+  mkdir -p /home/archivebox/.config
+  chown -R archivebox:archivebox /opt/archivebox /home/archivebox
   cd /opt/archivebox/data
-  $STD uv pip install --system --break-system-packages --upgrade --no-reinstall archivebox
-  sudo -u archivebox archivebox init
+  $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox init
+  base="$(grep -oP '^BASE_URL\s*=\s*\K\S+' ArchiveBox.conf 2>/dev/null || true)"
+  if [[ -z "$base" || "$base" =~ ^https?://[0-9.]+(:[0-9]+)?/?$ ]]; then
+    base="http://$(hostname -I | awk '{print $1}'):${port}"
+  fi
+  $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox config --set "BASE_URL=${base}" MERCURY_ENABLED=False
   msg_ok "Updated ArchiveBox"
 
+  msg_info "Installing Extractors (Patience)"
+  runtime="/tmp/archivebox-runtime-$(id -u archivebox)"
+  install -d -m 700 -o archivebox -g archivebox "$runtime"
+  if $STD env HOME=/home/archivebox USER=archivebox LOGNAME=archivebox XDG_CONFIG_HOME=/home/archivebox/.config XDG_RUNTIME_DIR="$runtime" \
+    /opt/archivebox/venv/bin/archivebox install; then
+    msg_ok "Installed Extractors"
+  else
+    msg_warn "Some extractors failed to install - crawls stop until the next update succeeds"
+  fi
+
+  if [[ "$migrate" == 1 ]]; then
+    msg_info "Migrating Existing Snapshots (Patience)"
+    $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox update --rescan --migrate-only --index-only
+    msg_ok "Migrated Existing Snapshots"
+  fi
+
   msg_info "Starting Service"
   systemctl start archivebox
   msg_ok "Started Service"
@@ -65,4 +114,4 @@ description
 msg_ok "Completed successfully!\n"
 echo -e "${CREATING}${GN}${APP} setup has been successfully initialized!${CL}"
 echo -e "${INFO}${YW}Access it using the following URL:${CL}"
-echo -e "${GATEWAY}${BGN}http://${IP}:8000/admin/login${CL}"
+echo -e "${GATEWAY}${BGN}http://${IP}:5797/admin/${CL}"

install/archivebox-install.sh

@@ -16,61 +16,41 @@ update_os
 msg_info "Installing Dependencies"
 $STD apt-get install -y \
   git \
-  expect \
-  libssl-dev \
-  libldap2-dev \
-  libsasl2-dev \
   procps \
   dnsutils \
-  ripgrep \
-  chromium
+  python3.13
 msg_ok "Installed Dependencies"
 
-msg_info "Installing Python Dependencies"
-$STD apt-get install -y \
-  python3-ldap \
-  python3-msgpack \
-  python3-regex
-msg_ok "Installed Python Dependencies"
-
-NODE_VERSION="22" NODE_MODULE="@postlight/parser@latest,single-file-cli@latest" setup_nodejs
-PYTHON_VERSION="3.13" setup_uv
-
-msg_info "Installing Playwright"
-$STD uv pip install playwright --system --break-system-packages
-$STD playwright install-deps chromium
-msg_ok "Installed Playwright"
+setup_uv
 
 msg_info "Installing ArchiveBox"
-mkdir -p /opt/archivebox/{data,.npm,.cache,.local}
+mkdir -p /opt/archivebox/data
 $STD adduser --system --shell /bin/bash --gecos 'Archive Box User' --group --disabled-password --home /home/archivebox archivebox
-chown -R archivebox:archivebox /opt/archivebox/{data,.npm,.cache,.local}
-chmod -R 755 /opt/archivebox/data
-$STD uv pip install archivebox --system --break-system-packages
-cd /opt/archivebox/data
-expect <<EOF
-set timeout -1
-log_user 0
-
-spawn sudo -u archivebox playwright install chromium
-spawn sudo -u archivebox archivebox setup
-
-expect "Username"
-send "\r"
-
-expect "Email address"
-send "\r"
-
-expect "Password"
-send "community-scripts.org\r"
-
-expect "Password (again)"
-send "community-scripts.org\r"
-
-expect eof
-EOF
+$STD uv venv --python /usr/bin/python3.13 /opt/archivebox/venv
+$STD uv pip install --python /opt/archivebox/venv/bin/python archivebox
+ln -sf /opt/archivebox/venv/bin/archivebox /usr/local/bin/archivebox
+mkdir -p /home/archivebox/.config
+chown -R archivebox:archivebox /opt/archivebox /home/archivebox
 msg_ok "Installed ArchiveBox"
 
+msg_info "Initializing ArchiveBox"
+cd /opt/archivebox/data
+$STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox init
+$STD sudo -u archivebox env \
+  DJANGO_SUPERUSER_USERNAME=archivebox \
+  DJANGO_SUPERUSER_EMAIL=archivebox@archivebox.local \
+  DJANGO_SUPERUSER_PASSWORD=community-scripts.org \
+  /opt/archivebox/venv/bin/archivebox manage createsuperuser --noinput
+$STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox config --set "BASE_URL=http://$(get_ip):5797" MERCURY_ENABLED=False
+msg_ok "Initialized ArchiveBox"
+
+msg_info "Installing Extractors (Patience)"
+AB_RUNTIME="/tmp/archivebox-runtime-$(id -u archivebox)"
+install -d -m 700 -o archivebox -g archivebox "$AB_RUNTIME"
+$STD env HOME=/home/archivebox USER=archivebox LOGNAME=archivebox XDG_CONFIG_HOME=/home/archivebox/.config XDG_RUNTIME_DIR="$AB_RUNTIME" \
+  /opt/archivebox/venv/bin/archivebox install
+msg_ok "Installed Extractors"
+
 msg_info "Creating Service"
 cat <<EOF >/etc/systemd/system/archivebox.service
 [Unit]
@@ -80,7 +60,8 @@ After=network.target
 [Service]
 User=archivebox
 WorkingDirectory=/opt/archivebox/data
-ExecStart=/usr/local/bin/archivebox server 0.0.0.0:8000
+ExecStartPre=/opt/archivebox/venv/bin/archivebox init
+ExecStart=/opt/archivebox/venv/bin/archivebox server 0.0.0.0:5797
 Restart=always
 
 [Install]