There are a number of open issues regarding problems with Vultr support. Can you take a look and see if you can diagnose them? The necessary environment variables to run the Vultr end to end tests have been configured on this session.
61 OPEN Vultr CI test fails 2026-03-16T21:02:07Z 60 OPEN Check if GH actions need updated 2026-03-16T20:57:57Z 57 OPEN Vultr machines aren't cleaned up in e2e test 2026-03-16T15:47:22Z 56 OPEN Address dependabot alerts 2026-03-16T15:41:25Z 55 OPEN Vultr DNS provisioning doesn't work 2026-03-16T15:38:35Z 53 OPEN Vultr CI tests fail 2026-03-16T15:02:43Z
1→import base64
2→
3→from vultr import Vultr, VultrException
4→
5→from machine.log import fatal_error, info
6→from machine.provider import CloudProvider, VM, SSHKey, DNSRecord
7→
8→
9→VALID_REGIONS = [
10→ "ewr", "ord", "dfw", "sea", "lax", "atl", "ams", "lhr", "fra",
11→ "sjc", "syd", "nrt", "cdg", "icn", "mia", "sgp", "sto",
12→ "mex", "mad", "sao", "del", "hnl", "yto", "blr", "jnb",
13→ "bom", "tlv",
14→]
15→
16→
17→def _instance_to_vm(instance) -> VM:
18→ return VM(
19→ id=instance.get("id", ""),
20→ name=instance.get("label", ""),
21→ tags=instance.get("tags", []),
22→ region=instance.get("region", ""),
23→ ip_address=instance.get("main_ip", ""),
24→ status=instance.get("status", ""),
25→ )
26→
27→
28→class VultrProvider(CloudProvider):
29→ def __init__(self, provider_config):
30→ info("WARNING: Vultr support is experimental and has not been fully verified. Use with caution.")
31→ if "api-key" not in provider_config:
32→ fatal_error("Required key 'api-key' not found in 'vultr' section of config file")
33→ self._api_key = provider_config["api-key"]
34→ self._client = Vultr(self._api_key)
35→
36→ def create_vm(self, name, region, image, size, ssh_key_name, tags, user_data) -> VM:
37→ ssh_key = self._get_vultr_ssh_key(ssh_key_name)
38→ if not ssh_key:
39→ fatal_error(f"Error: SSH key '{ssh_key_name}' not found in Vultr")
40→
41→ kwargs = {
42→ "os_id": int(image),
43→ "label": name,
44→ "hostname": name,
45→ "sshkey_id": [ssh_key["id"]],
46→ "tags": tags,
47→ "backups": "disabled",
48→ }
49→ if user_data:
50→ kwargs["user_data"] = base64.b64encode(user_data.encode()).decode()
51→
52→ try:
53→ result = self._client.create_instance(region, size, **kwargs)
54→ except VultrException as e:
55→ fatal_error(f"Error creating instance: {e}")
56→
57→ return _instance_to_vm(result)
58→
59→ def get_vm(self, vm_id) -> VM:
60→ try:
61→ result = self._client.get_instance(vm_id)
62→ except VultrException as e:
63→ fatal_error(f"Error: machine with id {vm_id} not found: {e}")
64→ return _instance_to_vm(result)
65→
66→ def destroy_vm(self, vm_id) -> bool:
67→ try:
68→ self._client.delete_instance(vm_id)
69→ except VultrException as e:
70→ fatal_error(f"Error: machine with id {vm_id} not found: {e}")
71→ return True
72→
73→ def list_vms(self, tag=None) -> list:
74→ try:
75→ params = {"tag": tag} if tag else None
76→ result = self._client.list_instances(params=params)
77→ except VultrException as e:
78→ fatal_error(f"Error listing instances: {e}")
79→ return [_instance_to_vm(i) for i in result]
80→
81→ def get_ssh_key(self, name) -> SSHKey:
82→ key = self._get_vultr_ssh_key(name)
83→ if not key:
84→ return None
85→ return SSHKey(
86→ id=key["id"],
87→ name=key["name"],
88→ fingerprint=key.get("fingerprint", ""),
89→ public_key=key.get("ssh_key", ""),
90→ )
91→
92→ def list_ssh_keys(self) -> list:
93→ try:
94→ result = self._client.list_keys()
95→ except VultrException as e:
96→ fatal_error(f"Error listing SSH keys: {e}")
97→ return [
98→ SSHKey(
99→ id=k["id"],
100→ name=k["name"],
101→ fingerprint=k.get("fingerprint", ""),
102→ public_key=k.get("ssh_key", ""),
103→ )
104→ for k in result
105→ ]
106→
107→ def create_dns_record(self, zone, record_type, name, data, ttl, tag=None) -> str:
108→ try:
109→ result = self._client.post(
110→ f"/domains/{zone}/records",
111→ type=record_type, name=name, data=data, ttl=ttl,
112→ )
113→ except VultrException:
114→ info(f"Warning: DNS zone '{zone}' not found in Vultr, DNS record not set")
115→ return None
116→ record = result.get("record", result)
117→ return record.get("id")
118→
119→ def delete_dns_record(self, zone, record_name) -> bool:
120→ records = self.get_dns_records(zone)
121→ for record in records:
122→ if record.name == record_name:
123→ try:
124→ self._client.delete(f"/domains/{zone}/records/{record.id}")
125→ except VultrException:
126→ return False
127→ return True
128→ return False
129→
130→ def get_dns_records(self, zone) -> list:
131→ try:
132→ result = self._client.get(f"/domains/{zone}/records")
133→ except VultrException:
134→ info(f"Warning: DNS zone '{zone}' not found in Vultr")
135→ return []
136→ records = result.get("records", [])
137→ return [
138→ DNSRecord(
139→ id=str(r.get("id", "")),
140→ name=r.get("name", ""),
141→ type=r.get("type", ""),
142→ data=r.get("data", ""),
143→ ttl=r.get("ttl", 0),
144→ )
145→ for r in records
146→ ]
147→
148→ def list_domains(self) -> list:
149→ try:
150→ result = self._client.get("/domains")
151→ except VultrException as e:
152→ fatal_error(f"Error listing domains: {e}")
153→ domains = result.get("domains", [])
154→ return [d.get("domain", "") for d in domains]
155→
156→ def validate_region(self, region):
157→ if region is not None and region.lower() not in VALID_REGIONS:
158→ fatal_error(f"Error: region {region} is not one of {VALID_REGIONS}")
159→
160→ def validate_image(self, image):
161→ try:
162→ int(image)
163→ except (ValueError, TypeError):
164→ info(f"Warning: Vultr image (os_id) should be a numeric ID. Got: {image}")
165→
166→ def _get_vultr_ssh_key(self, name):
167→ try:
168→ result = self._client.list_keys()
169→ except VultrException as e:
170→ fatal_error(f"Error listing SSH keys: {e}")
171→ for key in result:
172→ if key.get("name") == name:
173→ return key
174→ return None
175→
176→ @property
177→ def provider_name(self) -> str:
178→ return "Vultr"
179→gh issue view 61 && echo "---SEPARATOR---" && gh issue view 57 && echo "---SEPARATOR---" && gh issue view 55 && echo "---SEPARATOR---" && gh issue view 53
{
"subagent_type": "Explore",
"prompt": "Find all end-to-end test files related to Vultr in this project. Search for files matching patterns like *vultr*test*, *e2e*, *test*vultr*. Also search for any CI workflow files. Return the full paths and a summary of what each file does. Thoroughness: medium."
}title: Vultr CI test fails state: OPEN author: dboreham labels: comments: 0 assignees: projects: milestone: number: 61 -- This test began failing when the OS image id was changed from Debian to Ubuntu but it seems unlikely that's the root cause. Perhaps it's a timing thing and the timing changed a bit between Debian and Ubuntu, or just random chance. https://github.com/stirlingbridge/machine/actions/runs/23165517312/job/67304330705 ---SEPARATOR--- title: Vultr machines aren't cleaned up in e2e test state: OPEN author: dboreham labels: comments: 0 assignees: projects: milestone: number: 57 -- After each end to end test CI job runs the machine created (compute instance at Vultr) remains active. In the Digitalocean case the droplet is destroyed and they don't accumulate so there is some Vultr-specific difference here. ---SEPARATOR--- title: Vultr DNS provisioning doesn't work state: OPEN author: dboreham labels: comments: 0 assignees: projects: milestone: number: 55 -- Checking the status of DNS through the Vultr control panel, I noticed that although the DNS A record for the target VM had been created correctly in the zone, the value of the record was 0.0.0.0. This indicates that something about our end to end tests is not right. Test should be checking that not only is the DNS record created but that it has a valid value (the IP address of the created machine). It would also probably be worthwhile doing a full end to end check on this which would involve using the actual DNS name of the machine to make a connection to it that somehow checks that it's the right machine (e.g. check the host name by making an ssh connection then running the `hostname` command). ---SEPARATOR--- title: Vultr CI tests fail state: OPEN author: dboreham labels: comments: 0 assignees: projects: milestone: number: 53 --
[
{
"type": "text",
"text": "Perfect! Now I have gathered all the relevant information. Let me provide a comprehensive summary of the end-to-end test files and CI workflows related to Vultr.\n\n## Summary of End-to-End Test Files and CI Workflows\n\n### End-to-End Test Files\n\n1. **`/home/david/projects/bpi/machine/tests/test_e2e.py`**\n - Main end-to-end test file that creates real VMs on cloud providers\n - Contains configurable provider selection via `E2E_PROVIDER` environment variable (supports \"digital-ocean\" and \"vultr\")\n - Includes Vultr-specific defaults: region=\"ewr\", image=\"2284\", size=\"vc2-1c-1gb\"\n - Defines Vultr environment variable requirements: `E2E_VULTR_API_KEY`, `E2E_VULTR_DNS_ZONE`\n - Contains a `TestInstanceLifecycle` class with multiple test methods covering:\n - Instance listing and appearance\n - IP address assignment\n - DNS record creation and verification\n - Machine type tagging (cloud-init)\n - Custom tag application and filtering\n - Uses pytest fixtures for config file generation, session management, and instance lifecycle\n - Handles both numeric IDs (DigitalOcean) and UUID IDs (Vultr) in the `_extract_instance_id()` helper function\n\n2. **`/home/david/projects/bpi/machine/tests/E2E.md`**\n - Documentation file explaining end-to-end test setup and execution\n - Contains Vultr-specific prerequisites, environment variables, and configuration details\n - Notes important Vultr limitation: only supports second-level domains (e.g., `example.com`, not subdomains)\n - Lists default values for Vultr: region=\"ewr\", image=\"2136\" (Ubuntu 24.04), size=\"vc2-1c-1gb\"\n - Provides instructions for running tests locally against Vultr\n - Documents what is tested: instance lifecycle, DNS lifecycle, cloud-init initialization, custom tags\n\n3. **`/home/david/projects/bpi/machine/.claude/projects/-home-david-projects-stirlingbridge-machine/memory/project_e2e_tests.md`**\n - Project memory/decision document describing CI trigger strategy for e2e tests\n - Notes that e2e tests are deliberately limited to nightly schedule and manual triggers only\n - Explains decision rationale: runtime and cost considerations for real cloud resources\n\n### CI Workflow Files\n\n1. **`/home/david/projects/bpi/machine/.github/workflows/e2e-test.yml`**\n - Primary workflow for running end-to-end tests\n - Triggers: push to main, pull requests to main, manual dispatch, and weekly schedule (Mondays at 06:00 UTC)\n - Uses a matrix strategy to test both \"digital-ocean\" and \"vultr\" providers in parallel\n - Each job:\n - Checks out the repository\n - Installs uv and Python 3.13\n - Syncs dependencies\n - Runs pytest with e2e marker: `uv run pytest tests/test_e2e.py -v -m e2e`\n - Requires environment secrets: `E2E_VULTR_API_KEY` (for Vultr)\n - Requires environment variables: `E2E_SSH_KEY`, `E2E_VULTR_DNS_ZONE` (for Vultr)\n\n2. **`/home/david/projects/bpi/machine/.github/workflows/test.yml`**\n - Unit and integration test workflow (does NOT include e2e tests)\n - Triggers: push to main, pull requests to main\n - Runs standard pytest: `uv run pytest tests/ -v`\n - E2E tests are automatically skipped via pytest marker configuration\n\n3. **`/home/david/projects/bpi/machine/.github/workflows/build-release.yml`**\n - Build and release workflow\n - Triggers: push to main or build-test branches\n - Builds shiv executable package and publishes as GitHub release\n - Not directly related to testing, but part of the overall CI/CD pipeline"
},
{
"type": "text",
"text": "agentId: a9e85f4fbc0c50eb0 (for resuming to continue this agent's work if needed)\n<usage>total_tokens: 24758\ntool_uses: 15\nduration_ms: 17324</usage>"
}
] 1→"""End-to-end tests that create real VMs on a cloud provider.
2→
3→These tests require a real cloud provider environment and are NOT run as part of
4→the normal test suite. They must be invoked explicitly:
5→
6→ uv run pytest tests/test_e2e.py -v
7→
8→Provider selection:
9→ E2E_PROVIDER - Provider name: "digital-ocean" (default) or "vultr"
10→
11→Required environment variables (all providers):
12→ E2E_SSH_KEY - Name of an SSH key already registered with the provider
13→Required environment variables (DigitalOcean):
14→ E2E_DO_TOKEN - DigitalOcean API token
15→ E2E_DO_DNS_ZONE - DNS zone hosted at DigitalOcean (e.g. "do.example.com")
16→ E2E_PROJECT - DO project name to assign droplets to
17→
18→Required environment variables (Vultr):
19→ E2E_VULTR_API_KEY - Vultr API key
20→ E2E_VULTR_DNS_ZONE - DNS zone hosted at Vultr (e.g. "example.com")
21→
22→Optional environment variables:
23→ E2E_REGION - Region slug (default: provider-specific)
24→ E2E_IMAGE - Image slug or ID (default: provider-specific)
25→ E2E_SIZE - Machine size slug (default: provider-specific)
26→"""
27→
28→import json
29→import os
30→import subprocess
31→import uuid
32→
33→import pytest
34→
35→
36→# ---------------------------------------------------------------------------
37→# Provider configuration
38→# ---------------------------------------------------------------------------
39→
40→E2E_PROVIDER = os.environ.get("E2E_PROVIDER", "digital-ocean")
41→
42→_PROVIDER_DEFAULTS = {
43→ "digital-ocean": {
44→ "region": "nyc1",
45→ "image": "ubuntu-24-04-x64",
46→ "size": "s-1vcpu-512mb-10gb",
47→ },
48→ "vultr": {
49→ "region": "ewr",
50→ "image": "2284",
51→ "size": "vc2-1c-1gb",
52→ },
53→}
54→
55→_defaults = _PROVIDER_DEFAULTS.get(E2E_PROVIDER, _PROVIDER_DEFAULTS["digital-ocean"])
56→
57→E2E_SSH_KEY = os.environ.get("E2E_SSH_KEY")
58→
59→# Per-provider DNS zones
60→E2E_DO_DNS_ZONE = os.environ.get("E2E_DO_DNS_ZONE")
61→E2E_VULTR_DNS_ZONE = os.environ.get("E2E_VULTR_DNS_ZONE")
62→
63→# Select the DNS zone for the active provider
64→if E2E_PROVIDER == "digital-ocean":
65→ E2E_DNS_ZONE = E2E_DO_DNS_ZONE
66→elif E2E_PROVIDER == "vultr":
67→ E2E_DNS_ZONE = E2E_VULTR_DNS_ZONE
68→else:
69→ E2E_DNS_ZONE = None
70→E2E_REGION = os.environ.get("E2E_REGION", _defaults["region"])
71→E2E_IMAGE = os.environ.get("E2E_IMAGE", _defaults["image"])
72→E2E_SIZE = os.environ.get("E2E_SIZE", _defaults["size"])
73→
74→# Provider-specific credentials
75→E2E_DO_TOKEN = os.environ.get("E2E_DO_TOKEN")
76→E2E_PROJECT = os.environ.get("E2E_PROJECT")
77→E2E_VULTR_API_KEY = os.environ.get("E2E_VULTR_API_KEY")
78→
79→
80→# ---------------------------------------------------------------------------
81→# Skip the entire module if credentials are not provided
82→# ---------------------------------------------------------------------------
83→
84→pytestmark = pytest.mark.e2e
85→
86→_MISSING = []
87→if not E2E_SSH_KEY:
88→ _MISSING.append("E2E_SSH_KEY")
89→if E2E_PROVIDER == "digital-ocean":
90→ if not E2E_DO_TOKEN:
91→ _MISSING.append("E2E_DO_TOKEN")
92→ if not E2E_DO_DNS_ZONE:
93→ _MISSING.append("E2E_DO_DNS_ZONE")
94→ if not E2E_PROJECT:
95→ _MISSING.append("E2E_PROJECT")
96→elif E2E_PROVIDER == "vultr":
97→ if not E2E_VULTR_API_KEY:
98→ _MISSING.append("E2E_VULTR_API_KEY")
99→ if not E2E_VULTR_DNS_ZONE:
100→ _MISSING.append("E2E_VULTR_DNS_ZONE")
101→else:
102→ _MISSING.append(f"E2E_PROVIDER (unknown provider: {E2E_PROVIDER})")
103→
104→if _MISSING:
105→ pytestmark = [
106→ pytest.mark.e2e,
107→ pytest.mark.skip(reason=f"E2E env vars not set: {', '.join(_MISSING)}"),
108→ ]
109→
110→
111→# ---------------------------------------------------------------------------
112→# Helpers
113→# ---------------------------------------------------------------------------
114→
115→
116→def _unique_name(prefix="e2etest"):
117→ """Generate a short unique instance name safe for DNS."""
118→ return f"{prefix}-{uuid.uuid4().hex[:8]}"
119→
120→
121→def _write_config(path, **overrides):
122→ """Write a minimal config file for the e2e test run."""
123→ if E2E_PROVIDER == "digital-ocean":
124→ cfg = {
125→ "access-token": E2E_DO_TOKEN,
126→ "ssh-key": E2E_SSH_KEY,
127→ "dns-zone": E2E_DNS_ZONE,
128→ "machine-size": E2E_SIZE,
129→ "image": E2E_IMAGE,
130→ "region": E2E_REGION,
131→ "project": E2E_PROJECT,
132→ }
133→ cfg.update(overrides)
134→ provider_lines = "\n".join(f" {k}: {v}" for k, v in cfg.items())
135→ content = f"digital-ocean:\n{provider_lines}\nmachines:\n e2e-basic:\n new-user-name: e2euser\n"
136→ elif E2E_PROVIDER == "vultr":
137→ cfg = {
138→ "api-key": E2E_VULTR_API_KEY,
139→ "ssh-key": E2E_SSH_KEY,
140→ "dns-zone": E2E_DNS_ZONE,
141→ "machine-size": E2E_SIZE,
142→ "image": E2E_IMAGE,
143→ "region": E2E_REGION,
144→ }
145→ cfg.update(overrides)
146→ provider_lines = "\n".join(f" {k}: {v}" for k, v in cfg.items())
147→ content = f"vultr:\n{provider_lines}\nmachines:\n e2e-basic:\n new-user-name: e2euser\n"
148→
149→ with open(path, "w") as f:
150→ f.write(content)
151→
152→
153→def run_machine(*args, config_file=None, session_id=None):
154→ """Run the machine CLI as a subprocess with the given arguments."""
155→ cmd = ["uv", "run", "machine"]
156→ if config_file:
157→ cmd += ["--config-file", str(config_file)]
158→ if session_id:
159→ cmd += ["--session-id", session_id]
160→ cmd += list(args)
161→ result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
162→ return result
163→
164→
165→def _extract_instance_id(output_text):
166→ """Extract the instance ID from CLI output like 'New droplet created with id: 12345'.
167→
168→ Handles both numeric IDs (DigitalOcean) and UUID IDs (Vultr).
169→ """
170→ for line in output_text.splitlines():
171→ if "id:" in line.lower():
172→ parts = line.split("id:")
173→ if len(parts) >= 2:
174→ candidate = parts[-1].strip()
175→ if candidate:
176→ return candidate
177→ return None
178→
179→
180→# ---------------------------------------------------------------------------
181→# Fixtures
182→# ---------------------------------------------------------------------------
183→
184→
185→@pytest.fixture(scope="module")
186→def config_file(tmp_path_factory):
187→ """Write a config file that lives for the whole test module."""
188→ path = tmp_path_factory.mktemp("e2e") / "config.yml"
189→ _write_config(path)
190→ return path
191→
192→
193→@pytest.fixture(scope="module")
194→def session_id():
195→ """A unique session id shared across all tests in this module."""
196→ return uuid.uuid4().hex[:8]
197→
198→
199→@pytest.fixture(scope="class")
200→def instance(config_file, session_id):
201→ """Create a single instance with all features and destroy it after all tests.
202→
203→ The instance is created with DNS, a machine type (cloud-init), a custom tag,
204→ and --wait-for-ip so that all aspects can be verified by individual tests.
205→ """
206→ name = _unique_name()
207→ custom_tag = f"e2e-tag-{uuid.uuid4().hex[:6]}"
208→
209→ # ---- CREATE with all features ------------------------------------------
210→ result = run_machine(
211→ "create",
212→ "--name",
213→ name,
214→ "--type",
215→ "e2e-basic",
216→ "--update-dns",
217→ "--tag",
218→ custom_tag,
219→ "--wait-for-ip",
220→ config_file=config_file,
221→ session_id=session_id,
222→ )
223→ assert result.returncode == 0, f"create failed: {result.stderr}"
224→ create_out = result.stdout + result.stderr
225→ instance_id = _extract_instance_id(create_out)
226→ assert instance_id, f"Could not find instance id in output:\n{create_out}"
227→
228→ info = {
229→ "name": name,
230→ "id": instance_id,
231→ "custom_tag": custom_tag,
232→ "create_out": create_out,
233→ }
234→
235→ yield info
236→
237→ # ---- TEARDOWN: destroy with DNS cleanup --------------------------------
238→ run_machine(
239→ "--verbose",
240→ "destroy",
241→ "--no-confirm",
242→ "--delete-dns",
243→ instance_id,
244→ config_file=config_file,
245→ session_id=session_id,
246→ )
247→
248→
249→# ---------------------------------------------------------------------------
250→# Tests — one instance, many assertions
251→# ---------------------------------------------------------------------------
252→
253→
254→class TestInstanceLifecycle:
255→ """Create one instance with all features and verify each aspect independently.
256→
257→ A single instance is created (via the class-scoped ``instance`` fixture) with
258→ DNS, a machine type, and a custom tag. Each test method verifies a different
259→ aspect so that failures are reported individually. The instance is destroyed
260→ automatically after all tests complete.
261→ """
262→
263→ def test_instance_appears_in_list(self, instance, config_file, session_id):
264→ """Verify the instance shows up in ``list`` with the correct name."""
265→ result = run_machine(
266→ "list",
267→ "--output",
268→ "json",
269→ config_file=config_file,
270→ session_id=session_id,
271→ )
272→ assert result.returncode == 0, f"list failed: {result.stderr}"
273→ instances = json.loads(result.stdout)
274→ matched = [i for i in instances if str(i["id"]) == instance["id"]]
275→ assert len(matched) == 1, f"Expected 1 instance with id {instance['id']}, got {len(matched)}"
276→ assert matched[0]["name"] == instance["name"]
277→
278→ def test_instance_has_ip(self, instance, config_file, session_id):
279→ """Verify the instance was assigned an IP address."""
280→ result = run_machine(
281→ "list",
282→ "--output",
283→ "json",
284→ config_file=config_file,
285→ session_id=session_id,
286→ )
287→ assert result.returncode == 0
288→ instances = json.loads(result.stdout)
289→ matched = [i for i in instances if str(i["id"]) == instance["id"]]
290→ assert len(matched) == 1
291→ assert matched[0]["ip"] is not None, "Instance has no IP address"
292→
293→ def test_dns_record_created(self, instance, config_file, session_id):
294→ """Verify that a DNS A record was created for the instance."""
295→ result = run_machine(
296→ "list-domain",
297→ "--name",
298→ instance["name"],
299→ "--output",
300→ "json",
301→ E2E_DNS_ZONE,
302→ config_file=config_file,
303→ session_id=session_id,
304→ )
305→ assert result.returncode == 0, f"list-domain failed: {result.stderr}"
306→ records = json.loads(result.stdout)
307→ a_records = [r for r in records if r.get("name") == instance["name"] and r.get("type") == "A"]
308→ assert len(a_records) >= 1, f"No A record found for {instance['name']}.{E2E_DNS_ZONE}"
309→
310→ def test_dns_zone_in_create_output(self, instance):
311→ """Verify that DNS zone was mentioned in the create output."""
312→ assert E2E_DNS_ZONE in instance["create_out"], f"DNS zone not mentioned in output:\n{instance['create_out']}"
313→
314→ def test_type_tag_applied(self, instance, config_file, session_id):
315→ """Verify that the machine type tag was applied and is filterable."""
316→ result = run_machine(
317→ "list",
318→ "--type",
319→ "e2e-basic",
320→ "--output",
321→ "json",
322→ config_file=config_file,
323→ session_id=session_id,
324→ )
325→ assert result.returncode == 0
326→ instances = json.loads(result.stdout)
327→ matched = [i for i in instances if str(i["id"]) == instance["id"]]
328→ assert len(matched) == 1, "Instance not found when filtering by type e2e-basic"
329→ assert matched[0]["type"] == "e2e-basic", "Type tag mismatch"
330→
331→ def test_custom_tag_applied(self, instance, config_file, session_id):
332→ """Verify that the custom tag was applied and is filterable."""
333→ result = run_machine(
334→ "list",
335→ "--tag",
336→ instance["custom_tag"],
337→ "--output",
338→ "json",
339→ config_file=config_file,
340→ session_id=session_id,
341→ )
342→ assert result.returncode == 0
343→ instances = json.loads(result.stdout)
344→ matched = [i for i in instances if str(i["id"]) == instance["id"]]
345→ assert len(matched) == 1, f"Instance not found with tag {instance['custom_tag']}"
346→ 1→name: E2E Test
2→
3→on:
4→ push:
5→ branches: [main]
6→ pull_request:
7→ branches: [main]
8→ workflow_dispatch:
9→ schedule:
10→ # Run weekly on Monday at 06:00 UTC
11→ - cron: "0 6 * * 1"
12→
13→jobs:
14→ e2e:
15→ name: "E2E Tests (${{ matrix.provider }})"
16→ runs-on: ubuntu-latest
17→ environment: e2e
18→ strategy:
19→ fail-fast: false
20→ matrix:
21→ provider: [digital-ocean, vultr]
22→ steps:
23→ - name: "Clone project repository"
24→ uses: actions/checkout@v4
25→ - name: "Install uv"
26→ uses: astral-sh/setup-uv@v4
27→ - name: "Install Python"
28→ run: uv python install 3.13
29→ - name: "Install dependencies"
30→ run: uv sync
31→ - name: "Run E2E tests"
32→ env:
33→ E2E_PROVIDER: ${{ matrix.provider }}
34→ E2E_SSH_KEY: ${{ vars.E2E_SSH_KEY }}
35→ # DigitalOcean credentials
36→ E2E_DO_TOKEN: ${{ secrets.E2E_DO_TOKEN }}
37→ E2E_DO_DNS_ZONE: ${{ vars.E2E_DO_DNS_ZONE }}
38→ E2E_PROJECT: ${{ vars.E2E_PROJECT }}
39→ # Vultr credentials
40→ E2E_VULTR_API_KEY: ${{ secrets.E2E_VULTR_API_KEY }}
41→ E2E_VULTR_DNS_ZONE: ${{ vars.E2E_VULTR_DNS_ZONE }}
42→ run: uv run pytest tests/test_e2e.py -v -m e2e
43→Let me look at the create/destroy subcommands and the factory to understand the full flow.
{
"subagent_type": "Explore",
"prompt": "Read these files and summarize the key logic in each, especially anything related to DNS, waiting for IP, and instance destruction:\n- machine/subcommands/ - all files in this directory\n- machine/factory.py\n- machine/provider.py\n- machine/di.py\nThoroughness: very thorough. Focus on how create handles --wait-for-ip, --update-dns, and how destroy handles --delete-dns. Also look at how the list command works with tags and JSON output."
}pip show vultr-python 2>/dev/null || uv run pip show vultr-python 2>/dev/null || uv pip show vultr-python 2>/dev/null
[
{
"type": "text",
"text": "Perfect! Now let me create a comprehensive summary. Let me organize the findings by file:\n\n## Summary of Key Logic\n\n### **machine/di.py**\nA simple dependency injection module using a class `d` with a static `opt` attribute (holds CliOptions). It's essentially a global options container used throughout the codebase.\n\n---\n\n### **machine/provider.py**\nAbstract base class `CloudProvider` defining the interface all providers must implement:\n\n**VM Operations:**\n- `create_vm()` - Creates a VM, returns VM object with id, name, tags, region, ip_address, status\n- `get_vm(vm_id)` - Retrieves a single VM by ID\n- `destroy_vm(vm_id)` - Deletes a VM, returns boolean success\n- `list_vms(tag=None)` - Lists VMs, optionally filtered by tag\n\n**SSH Key Management:**\n- `get_ssh_key(name)` - Gets SSH key by name\n- `list_ssh_keys()` - Lists all SSH keys\n\n**DNS Management:**\n- `create_dns_record(zone, record_type, name, data, ttl, tag=None)` - Creates A/AAAA DNS record, returns record ID\n- `delete_dns_record(zone, record_name)` - Deletes DNS record by zone and record name\n- `get_dns_records(zone)` - Lists all DNS records for a zone\n\n**Other:**\n- `list_domains()` - Lists available domains\n- `list_projects()` - Lists projects (returns empty list by default)\n- `assign_to_project(project_name, vm_id)` - Assigns VM to project (no-op by default)\n- `validate_region()` and `validate_image()` - Optional validation methods\n\n---\n\n### **machine/factory.py**\nMinimal file - just provides a `yaml()` function that instantiates a ruamel.yaml YAML object configured with:\n- `preserve_quotes = True`\n- Indent: sequence=3, offset=1\n\n---\n\n### **machine/types.py**\nDefines key data structures and constants:\n\n**Constants:**\n- `TAG_MACHINE_CREATED = \"machine:created\"` - Tag marking VMs created by this tool\n- `TAG_MACHINE_TYPE_PREFIX = \"machine:type:\"` - Prefix for machine type tags (e.g., \"machine:type:webapp\")\n- `TAG_MACHINE_SESSION_PREFIX = \"machine:session:\"` - Prefix for session ID tags (e.g., \"machine:session:abc123def\")\n\n**Dataclasses:**\n- `CliOptions(debug, quiet, verbose, dry_run)` - CLI global options\n- `Config(provider_name, provider_config, ssh_key, dns_zone, machine_size, image, region, project)` - Config from ~/.machine/config.yml\n- `MainCmdCtx(config, session_id, provider)` - Context passed to all commands\n- `MachineConfig(new_user_name, script_url, script_dir, script_path, script_args)` - Machine type definition\n\n---\n\n### **machine/subcommands/create.py**\nCreates a VM with optional DNS and IP waiting.\n\n**Key Options:**\n- `--wait-for-ip` (default False) - Wait until VM has an IP address\n- `--update-dns` (default True) - Create DNS A record after VM is created\n- `--initialize` (default True) - Use cloud-init to initialize the machine\n- `--name` (required) - VM name\n- `--type` - Machine type (required if initializing)\n- `--region`, `--machine-size`, `--image` - Override defaults from config\n- `--tag` - Additional tag to apply\n\n**Main Flow:**\n1. Validates DNS zone exists if `--update-dns` requested\n2. Gets user_data from cloud_config if `--initialize`\n3. Verifies SSH key exists\n4. Validates region and image if provided\n5. Creates tags: session tag, \"machine:created\" tag, machine type tag, custom tag\n6. Calls `provider.create_vm()` with all parameters\n7. Outputs VM ID (quiet) or detailed info\n8. Assigns to project if configured\n9. **If `--wait-for-ip` or `--update-dns` and no IP:**\n - Polls `provider.get_vm()` every 1 second until ip_address is set\n - Outputs \"Waiting for droplet IP address\" if verbose\n10. **If `--update-dns` and has IP:**\n - Creates A record with TTL=300 (5 minutes)\n - Tags record with \"machine:created\"\n - Outputs DNS record info\n\n---\n\n### **machine/subcommands/destroy.py**\nDestroys VMs with optional DNS cleanup.\n\n**Key Options:**\n- `--confirm` (default True) - Requires user to type \"YES\" to confirm\n- `--delete-dns` (default True) - Delete DNS record when VM is destroyed\n- `--all` (default False) - Allow destroying machines not created by this tool\n\n**Main Flow:**\n1. For each droplet_id argument:\n - Gets VM details\n - Checks if VM was created by machine tool (unless `--all`)\n - Checks if VM was created by same session (unless `--all`)\n - Prompts for confirmation if `--confirm` enabled\n - Calls `provider.destroy_vm(droplet_id)`\n - **If destroy succeeded and `--delete-dns` and dns_zone configured:**\n - Calls `provider.delete_dns_record(zone, vm_name)`\n - Logs debug messages about deletion\n - Fails if destroy operation returns false\n\n---\n\n### **machine/subcommands/list.py**\nLists VMs with flexible filtering and output formats.\n\n**Key Helper Function: `get_vms()`**\n- Filters by id (single VM), name, tag, type, region\n- By default returns only VMs from current session (filters by session tag)\n- Unless `--all`, filters to only \"machine:created\" tagged VMs from same session\n- Custom filters for tag, type (using TAG_MACHINE_TYPE_PREFIX), and region\n- Returns list of matching VMs\n\n**Output Formats:**\n- `print_normal()` - Shows: name (id, region, type): ip_address\n- `print_quiet()` - Shows only VM IDs\n- `print_json()` - Calls `vm_to_json_obj()` for each VM, outputs JSON array\n\n**Key Options:**\n- `--output json` - Output as JSON\n- `--quiet` - Output only IDs\n- `--tag`, `--type`, `--region`, `--name`, `--id` - Filter criteria\n- `--all` - Include machines not created by tool or other sessions\n- `--unique` - Error if more than one match\n\n---\n\n### **machine/subcommands/status.py**\nQueries machine status via HTTP requests.\n\n**Main Flow:**\n1. Uses `get_vms()` to filter machines\n2. For each VM:\n - Fetches status from `http://{vm.ip_address}:4242/cgi-bin/{status_check}`\n - Default status_check: \"cloud-init-status\"\n - Extracts `status` field from JSON response\n - Catches exceptions silently (sets status to \"UNKNOWN\")\n3. Returns dict with name, id, droplet-status, and check result\n\n**Output Formats:**\n- JSON or normal (tab-separated key=value pairs)\n\n---\n\n### **machine/subcommands/list_domain.py**\nLists DNS records for a zone.\n\n**Main Flow:**\n1. Gets zone from argument or config.dns_zone\n2. Calls `provider.get_dns_records(zone)`\n3. Filters by:\n - Record type (defaults to A and AAAA)\n - Name if specified\n4. Unless `--all`, filters to only records pointing to VMs from current session\n5. In JSON mode, links DNS records to matching VMs by IP address\n\n**Output Formats:**\n- `print_normal()` - name, type, data (tab-separated)\n- `print_quiet()` - record names only\n- `print_json()` - Calls `dns_record_to_json_obj()` with zone and matching VM\n\n---\n\n### **machine/subcommands/ssh_keys.py**\nSimple list command - displays all SSH keys in format: id: name (fingerprint)\n\n---\n\n### **machine/subcommands/projects.py**\nSimple list command - displays all projects returned by provider\n\n---\n\n### **machine/subcommands/domains.py**\nSimple list command - displays all domains returned by provider\n\n---\n\n### **machine/util.py**\nUtility helper functions:\n\n- `get_machine_type(vm)` - Extracts machine type from TAG_MACHINE_TYPE_PREFIX tag\n- `is_machine_created(vm)` - Checks if TAG_MACHINE_CREATED tag present\n- `is_same_session(context, vm)` - Checks if VM has session tag matching current session\n- `load_session_id()` - Loads or creates session ID from ~/.machine/session.yml\n- `vm_to_json_obj(vm)` - Converts VM to JSON object with id, name, tags, region, ip (from ip_address), type\n- `dns_record_to_json_obj(record, zone, vm)` - Converts DNS record to JSON with id, droplet (full VM object or null), name, fqdn, zone, data, ttl, type\n\n---\n\n## Key Architectural Patterns\n\n1. **Tagging System**: VMs are tagged with machine:created, machine:session:XXX, and machine:type:XXX to track provenance and enable filtering\n2. **Session-based Filtering**: By default, commands only operate on machines created in the current session (unless --all flag)\n3. **DNS Integration**: DNS records are created with TTL=300 and automatically cleaned up on destroy (unless --no-delete-dns)\n4. **IP Polling**: The create command polls every 1 second until the IP address is assigned by the cloud provider\n5. **JSON Output**: Multiple commands support --output json for programmatic consumption\n6. **Provider Abstraction**: All cloud provider logic is abstracted behind CloudProvider interface\n\n---\n\n## File Paths Reference\n\n- `/home/david/projects/bpi/machine/machine/di.py` - Dependency injection/globals\n- `/home/david/projects/bpi/machine/machine/provider.py` - Abstract provider interface\n- `/home/david/projects/bpi/machine/machine/factory.py` - YAML factory\n- `/home/david/projects/bpi/machine/machine/types.py` - Data structures and constants\n- `/home/david/projects/bpi/machine/machine/util.py` - Helper utilities\n- `/home/david/projects/bpi/machine/machine/subcommands/create.py` - Create VM command\n- `/home/david/projects/bpi/machine/machine/subcommands/destroy.py` - Destroy VM command\n- `/home/david/projects/bpi/machine/machine/subcommands/list.py` - List VMs command\n- `/home/david/projects/bpi/machine/machine/subcommands/status.py` - Status check command\n- `/home/david/projects/bpi/machine/machine/subcommands/list_domain.py` - List DNS records command\n- `/home/david/projects/bpi/machine/machine/subcommands/ssh_keys.py` - List SSH keys command\n- `/home/david/projects/bpi/machine/machine/subcommands/projects.py` - List projects command\n- `/home/david/projects/bpi/machine/machine/subcommands/domains.py` - List domains command"
},
{
"type": "text",
"text": "agentId: ab99f5e0d90eb295f (for resuming to continue this agent's work if needed)\n<usage>total_tokens: 26308\ntool_uses: 19\nduration_ms: 33992</usage>"
}
]Name: vultr-python Version: 0.2.0 Location: /home/david/projects/bpi/machine/.venv/lib/python3.12/site-packages Requires: requests Required-by: machine
Let me look at the vultr-python library API and the specific areas most likely to have bugs.
python3 -c "import vultr; import inspect; print(inspect.getsource(vultr.Vultr.list_instances))"
python3 -c "import vultr; import inspect; print(inspect.getsource(vultr.Vultr.create_instance))"
python3 -c "import vultr; import inspect; print(inspect.getsource(vultr.Vultr.get_instance))"
Exit code 1 Traceback (most recent call last): File "<string>", line 1, in <module> ModuleNotFoundError: No module named 'vultr'
<tool_use_error>Cancelled: parallel tool call Bash(python3 -c "import vultr; import inspect…) errored</tool_use_error>
<tool_use_error>Cancelled: parallel tool call Bash(python3 -c "import vultr; import inspect…) errored</tool_use_error>
uv run python3 -c "import vultr; import inspect; print(inspect.getsource(vultr.Vultr))" 2>&1 | head -200
class Vultr(object):
url = "https://api.vultr.com/v2"
def __init__(self, api_key: Optional[str] = None):
""":param api_key: Vultr API Key or `VULTR_API_KEY` Environment Variable"""
self.api_key = api_key or os.getenv("VULTR_API_KEY")
"""Provide the API key here or with the `VULTR_API_KEY` environment variable"""
self._session = requests.session()
if self.api_key:
self._session.headers.update({"Authorization": f"Bearer {self.api_key}"})
def get(self, url: str, params: Optional[dict] = None) -> Any:
"""
GET Data
:param url: Request URL. Example `/instances`
:param params: Query Parameters Dictionary
:return: Response Data
:raises: `VultrException`
"""
return self._req("get", f"{self.url}/{url.lstrip('/')}", params=params)
def post(self, url: str, **kwargs) -> Any:
"""
POST Data
:param url: Request URL. Example `/instances`
:param kwargs: Request Data Keyword Arguments
:return: Response Data
:raises: `VultrException`
"""
return self._req("post", f"{self.url}/{url.lstrip('/')}", kwargs)
def patch(self, url: str, **kwargs) -> Any:
"""
PATCH Data
:param url: Request URL. Example `/instances/{instance-id}`
:param kwargs: Request Data Keyword Arguments
:return: Response Data
:raises: `VultrException`
"""
return self._req("patch", f"{self.url}/{url.lstrip('/')}", kwargs)
def put(self, url: str, **kwargs) -> Any:
"""
PUT Data
:param url: Request URL. Example `/instances/{instance-id}`
:param kwargs: Request Data Keyword Arguments
:return: Response Data
:raises: `VultrException`
"""
return self._req("put", f"{self.url}/{url.lstrip('/')}", kwargs)
def delete(self, url: str) -> None:
"""
DELETE a Resource
:param url: Request URL. Example `/instances/{instance-id}`
:return: None
:raises: `VultrException`
"""
return self._req("delete", f"{self.url}/{url.lstrip('/')}")
def list_os(self, params: Optional[dict] = None) -> list:
url = f"{self.url}/os"
return self._req("get", url, params=params)["os"]
def list_plans(self, params: Optional[dict] = None) -> list:
url = f"{self.url}/plans"
return self._req("get", url, params=params)["plans"]
def list_regions(self, params: Optional[dict] = None) -> list:
url = f"{self.url}/regions"
return self._req("get", url, params=params)["regions"]
def list_instances(self, params: Optional[dict] = None) -> list:
url = f"{self.url}/instances"
return self._req("get", url, params=params)["instances"]
def get_instance(self, instance: Union[str, dict], params: Optional[dict] = None) -> dict:
instance_id = self._get_obj_key(instance)
url = f"{self.url}/instances/{instance_id}"
return self._req("get", url, params=params)["instance"]
def create_instance(self, region: Union[str, dict], plan: Union[str, dict], **kwargs) -> dict:
data = {"region": self._get_obj_key(region), "plan": self._get_obj_key(plan)}
data.update(kwargs)
url = f"{self.url}/instances"
return self._req("post", url, data)["instance"]
def update_instance(self, instance: Union[str, dict], **kwargs) -> dict:
instance_id = self._get_obj_key(instance)
url = f"{self.url}/instances/{instance_id}"
return self._req("patch", url, kwargs)["instance"]
def delete_instance(self, instance: Union[str, dict]) -> None:
instance_id = self._get_obj_key(instance)
url = f"{self.url}/instances/{instance_id}"
return self._req("delete", url)
def list_keys(self, params: Optional[dict] = None) -> list:
url = f"{self.url}/ssh-keys"
return self._req("get", url, params=params)["ssh_keys"]
def get_key(self, key: Union[str, dict], params: Optional[dict] = None) -> dict:
key_id = self._get_obj_key(key)
url = f"{self.url}/ssh-keys/{key_id}"
return self._req("get", url, params=params)["ssh_key"]
def create_key(self, name: str, key: str, **kwargs) -> dict:
data = {"name": name, "ssh_key": key}
data.update(kwargs)
url = f"{self.url}/ssh-keys"
return self._req("post", url, data)["ssh_key"]
def update_key(self, key: Union[str, dict], **kwargs) -> None:
key_id = self._get_obj_key(key)
url = f"{self.url}/ssh-keys/{key_id}"
return self._req("patch", url, kwargs)["ssh_key"]
def delete_key(self, key: Union[str, dict]) -> None:
key_id = self._get_obj_key(key)
url = f"{self.url}/ssh-keys/{key_id}"
return self._req("delete", url)
def list_scripts(self, params: Optional[dict] = None) -> list:
url = f"{self.url}/startup-scripts"
return self._req("get", url, params=params)["startup_scripts"]
def get_script(self, script: Union[str, dict], params: Optional[dict] = None) -> dict:
script_id = self._get_obj_key(script)
url = f"{self.url}/startup-scripts/{script_id}"
return self._req("get", url, params=params)["startup_script"]
def create_script(self, name: str, script: str, **kwargs) -> dict:
data = {"name": name, "script": script}
data.update(kwargs)
url = f"{self.url}/startup-scripts"
return self._req("post", url, data)["startup_script"]
def update_script(self, script: Union[str, dict], **kwargs) -> None:
script_id = self._get_obj_key(script)
url = f"{self.url}/startup-scripts/{script_id}"
return self._req("patch", url, kwargs)["startup_script"]
def delete_script(self, script: Union[str, dict]) -> None:
script_id = self._get_obj_key(script)
url = f"{self.url}/startup-scripts/{script_id}"
return self._req("delete", url)
def list_ipv4(self, instance: Union[str, dict], params: Optional[dict] = None) -> list:
instance_id = self._get_obj_key(instance)
url = f"{self.url}/instances/{instance_id}/ipv4"
return self._req("get", url, params=params)["ipv4s"]
def create_ipv4(self, instance: Union[str, dict], **kwargs) -> dict:
instance_id = self._get_obj_key(instance)
url = f"{self.url}/instances/{instance_id}/ipv4"
return self._req("post", url, kwargs)["ipv4"]
def delete_ipv4(self, instance: Union[str, dict]) -> None:
instance_id = self._get_obj_key(instance)
url = f"{self.url}/instances/{instance_id}/ipv4"
return self._req("delete", url)
@staticmethod
def filter_list(item_list: List[dict], value: str, key: str = "name") -> dict:
"""
Helper Function to get an Item from a List of Dictionaries
:param item_list: List to filter
:param value: Value of the Key
:param key: Key to check for Value
:return: Item or {}
"""
return next((d for d in item_list if str(d.get(key, "")).lower() == value.lower()), {})
@staticmethod
def filter_regions(regions: list, locations: list) -> list:
return [d for d in regions if d["id"] in locations]
@staticmethod
def filter_keys(keys: list, name: str) -> dict:
"""Soft Deprecated in 0.2.0. Use `Vultr.filter_list()`"""
warnings.warn("Soft Deprecated in 0.2.0. Use filter_list()", PendingDeprecationWarning, stacklevel=2)
try:
return next(d for d in keys if d["name"].lower() == name.lower())
except StopIteration:
return {}
@staticmethod
def filter_os(os_list: list, name: str) -> dict:
"""Soft Deprecated in 0.2.0. Use `Vultr.filter_list()`"""
warnings.warn("Soft Deprecated in 0.2.0. Use filter_list()", PendingDeprecationWarning, stacklevel=2)
try:
return next(d for d in os_list if d["name"].lower() == name.lower())
except StopIteration:
return {}
@staticmethod
def filter_scripts(scripts: list, name: str) -> dict:
"""Soft Deprecated in 0.2.0. Use `Vultr.filter_list()`"""
warnings.warn("Soft Deprecated in 0.2.0. Use filter_list()", PendingDeprecationWarning, stacklevel=2)
try: 1→import click
2→import time
3→from machine.config import get_machine
4→from machine.di import d
5→from machine.log import fatal_error, info, debug, output
6→from machine.types import MainCmdCtx, TAG_MACHINE_CREATED, TAG_MACHINE_TYPE_PREFIX
7→from machine.cloud_config import get_user_data
8→
9→from machine.types import TAG_MACHINE_SESSION_PREFIX
10→
11→
12→@click.command(help="Create a machine")
13→@click.option("--name", "-n", required=True, metavar="<MACHINE-NAME>", help="Name for new machine")
14→@click.option("--tag", "-t", metavar="<TAG-TEXT>", help="tag to be applied to new machine")
15→@click.option("--type", "-m", metavar="<MACHINE-TYPE>", help="create a machine of this type")
16→@click.option("--region", "-r", metavar="<REGION-CODE>", help="create a machine in this region (overrides default from config)")
17→@click.option(
18→ "--machine-size", "-s", metavar="<MACHINE-SLUG>", help="create a machine of this size (overrides default from config)"
19→)
20→@click.option("--image", "-s", metavar="<IMAGE-NAME>", help="create a machine from this image (overrides default from config)")
21→@click.option("--wait-for-ip/--no-wait-for-up", default=False)
22→@click.option("--update-dns/--no-update-dns", default=True)
23→@click.option("--initialize/--no-initialize", default=True)
24→@click.pass_context
25→def command(context, name, tag, type, region, machine_size, image, wait_for_ip, update_dns, initialize):
26→ command_context: MainCmdCtx = context.obj
27→ config = command_context.config
28→ provider = command_context.provider
29→
30→ if update_dns and not config.dns_zone:
31→ fatal_error("Error: DNS update requested but no zone configured")
32→
33→ user_data = None
34→ if initialize:
35→ if not type:
36→ fatal_error("Error: a machine type must be supplied")
37→ machine_config = get_machine(type)
38→ if not machine_config:
39→ fatal_error(f"Error: machine type {type} is not defined")
40→ fqdn = f"{name}.{config.dns_zone}" if config.dns_zone else None
41→ user_data = get_user_data(provider, config.ssh_key, fqdn, machine_config)
42→ if d.opt.debug:
43→ info("user-data is:")
44→ info(user_data)
45→
46→ # Verify SSH key exists
47→ ssh_key = provider.get_ssh_key(config.ssh_key)
48→ if not ssh_key:
49→ fatal_error(f"Error: SSH key '{config.ssh_key}' not found in {provider.provider_name}")
50→
51→ provider.validate_region(region)
52→ provider.validate_image(image)
53→
54→ tags = [
55→ TAG_MACHINE_SESSION_PREFIX + command_context.session_id,
56→ TAG_MACHINE_CREATED,
57→ ]
58→ if type:
59→ tags.append(TAG_MACHINE_TYPE_PREFIX + type.lower())
60→ if tag:
61→ tags.append(tag)
62→
63→ vm = provider.create_vm(
64→ name=name,
65→ region=region if region is not None else config.region,
66→ image=image if image is not None else config.image,
67→ size=machine_size if machine_size is not None else config.machine_size,
68→ ssh_key_name=config.ssh_key,
69→ tags=tags,
70→ user_data=user_data,
71→ )
72→
73→ if vm.id:
74→ if d.opt.quiet:
75→ output(f"{vm.id}")
76→ else:
77→ output(f"New droplet created with id: {vm.id}")
78→
79→ # If requested, assign to a specified project
80→ if config.project:
81→ provider.assign_to_project(config.project, vm.id)
82→ if d.opt.verbose:
83→ info(f"Assigned droplet to project: {config.project}")
84→
85→ # If requested, or if we are going to set a DNS record get the VM's IPv4 address
86→ ip_address = vm.ip_address
87→ if (wait_for_ip or update_dns) and not ip_address:
88→ while not ip_address:
89→ time.sleep(1)
90→ vm = provider.get_vm(vm.id)
91→ ip_address = vm.ip_address
92→ if d.opt.verbose:
93→ output("Waiting for droplet IP address")
94→ if d.opt.quiet:
95→ info(f"{ip_address}")
96→ else:
97→ info(f"IP Address: {ip_address}")
98→
99→ # If requested, and we have the IP address, create a DNS host record
100→ if update_dns and ip_address and config.dns_zone:
101→ zone = config.dns_zone
102→ host = name
103→ if d.opt.debug:
104→ debug(f"Setting host record {host}.{zone} to {ip_address}")
105→ record = provider.create_dns_record(
106→ zone=zone,
107→ record_type="A",
108→ name=host,
109→ data=ip_address,
110→ ttl=60 * 5,
111→ tag=TAG_MACHINE_CREATED,
112→ )
113→ if record:
114→ if d.opt.verbose:
115→ info(f"Created DNS record:{record}")
116→ if not d.opt.quiet:
117→ info(f"DNS: {host}.{zone}")
118→ 1→import click
2→
3→from machine.di import d
4→from machine.log import debug, fatal_error, output
5→from machine.util import is_machine_created
6→from machine.types import MainCmdCtx
7→
8→from machine.util import is_same_session
9→
10→
11→@click.command(help="Destroy one or more machines")
12→@click.option("--confirm/--no-confirm", default=True)
13→@click.option("--delete-dns/--no-delete-dns", default=True)
14→@click.option(
15→ "--all",
16→ is_flag=True,
17→ default=False,
18→ help="Include machines not created by this tool",
19→)
20→@click.argument("droplet-ids", nargs=-1)
21→@click.pass_context
22→def command(context, confirm, delete_dns, all, droplet_ids):
23→ command_context: MainCmdCtx = context.obj
24→ config = command_context.config
25→ provider = command_context.provider
26→
27→ for droplet_id in droplet_ids:
28→ vm = provider.get_vm(droplet_id)
29→ name = vm.name
30→
31→ if not is_machine_created(vm) and not all:
32→ fatal_error(f'ERROR: Cannot destroy droplet "{name}" (id: {vm.id}), it was not created by machine.')
33→
34→ if not is_same_session(command_context, vm) and not all:
35→ fatal_error(
36→ f'ERROR: Cannot destroy droplet "{name}" (id: {vm.id}), it was created by a different session of machine.'
37→ )
38→
39→ if confirm:
40→ output(
41→ "Type YES (not y or yes or Yes) to confirm that you want to permanently"
42→ f' DELETE/DESTROY droplet "{name}" (id: {vm.id})'
43→ )
44→ confirmation = input()
45→ if confirmation != "YES":
46→ fatal_error("Destroy operation aborted, not confirmed by user")
47→
48→ result = provider.destroy_vm(droplet_id)
49→
50→ if result and delete_dns and config.dns_zone:
51→ zone = config.dns_zone
52→ if d.opt.debug:
53→ debug(f"Deleting host record {name}.{zone}")
54→ deleted = provider.delete_dns_record(zone, name)
55→ if deleted:
56→ if d.opt.debug:
57→ debug(f"Deleted dns record for {name}.{zone}")
58→ else:
59→ if d.opt.debug:
60→ debug(f"No dns record found for {name}.{zone}")
61→
62→ if not result:
63→ fatal_error("Error destroying machine")
64→Now let me look at the Vultr list_instances to understand how tag filtering works, and check the IP address issue.
1→import click
2→import json
3→
4→from machine.log import fatal_error, output
5→from machine.types import MainCmdCtx, TAG_MACHINE_TYPE_PREFIX, TAG_MACHINE_SESSION_PREFIX
6→from machine.util import get_machine_type, is_machine_created, is_same_session, vm_to_json_obj
7→
8→
9→def print_normal(vms):
10→ for vm in vms:
11→ region = vm.region if vm.region else "unknown"
12→ output(f"{vm.name} ({vm.id}, {region}, {get_machine_type(vm)}): {vm.ip_address}")
13→
14→
15→def print_quiet(vms):
16→ for vm in vms:
17→ output(vm.id)
18→
19→
20→def print_json(vms):
21→ output(json.dumps([vm_to_json_obj(v) for v in vms]))
22→
23→
24→def get_vms(command_context, id=None, name=None, tag=None, type=None, region=None, all=False):
25→ provider = command_context.provider
26→
27→ vms = []
28→ if id:
29→ vm = provider.get_vm(id)
30→ if vm:
31→ vms.append(vm)
32→
33→ if all:
34→ vms = provider.list_vms()
35→ else:
36→ vms = provider.list_vms(tag=TAG_MACHINE_SESSION_PREFIX + command_context.session_id)
37→
38→ # we can't combine most filters over the API, so we also filter ourselves
39→ if name:
40→ vms = filter(lambda v: v.name == name, vms)
41→
42→ if tag:
43→ vms = filter(lambda v: tag in v.tags, vms)
44→
45→ if type:
46→ vms = filter(lambda v: TAG_MACHINE_TYPE_PREFIX + type.lower() in v.tags, vms)
47→
48→ if region:
49→ vms = filter(lambda v: v.region and region == v.region, vms)
50→
51→ if not all:
52→ vms = filter(lambda v: is_machine_created(v) and is_same_session(command_context, v), vms)
53→
54→ return list(vms)
55→
56→
57→@click.command(help="List machines")
58→@click.option("--id", metavar="<MACHINE-ID>", help="Filter by id")
59→@click.option("--name", "-n", metavar="<MACHINE-NAME>", help="Filter by name")
60→@click.option("--tag", "-t", metavar="<TAG-TEXT>", help="Filter by tag")
61→@click.option("--type", "-m", metavar="<MACHINE-TYPE>", help="Filter by type")
62→@click.option("--region", "-r", metavar="<REGION>", help="Filter by region")
63→@click.option("--output", "-o", metavar="<FORMAT>", help="Output format")
64→@click.option(
65→ "--all",
66→ is_flag=True,
67→ default=False,
68→ help="All machines, including those not created by this tool or by other sessions",
69→)
70→@click.option("--quiet", "-q", is_flag=True, default=False, help="Only display machine IDs")
71→@click.option(
72→ "--unique",
73→ is_flag=True,
74→ default=False,
75→ help="Return an error if there is more than one match",
76→)
77→@click.pass_context
78→def command(context, id, name, tag, type, region, all, output, quiet, unique):
79→ command_context: MainCmdCtx = context.obj
80→
81→ vms = get_vms(command_context, id, name, tag, type, region, all)
82→ if unique and len(vms) > 1:
83→ fatal_error(f"ERROR: --unique match required but {len(vms)} matches found.")
84→
85→ if output == "json":
86→ print_json(vms)
87→ elif quiet:
88→ print_quiet(vms)
89→ else:
90→ print_normal(vms)
91→Now I have a good picture of the codebase. Let me run the Vultr e2e tests to see the failures firsthand.
E2E_PROVIDER=vultr uv run pytest tests/test_e2e.py -v -m e2e 2>&1
Exit code 1
============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-9.0.2, pluggy-1.6.0 -- /home/david/projects/bpi/machine/.venv/bin/python3
cachedir: .pytest_cache
rootdir: /home/david/projects/bpi/machine
configfile: pyproject.toml
collecting ... collected 6 items
tests/test_e2e.py::TestInstanceLifecycle::test_instance_appears_in_list PASSED [ 16%]
tests/test_e2e.py::TestInstanceLifecycle::test_instance_has_ip PASSED [ 33%]
tests/test_e2e.py::TestInstanceLifecycle::test_dns_record_created FAILED [ 50%]
tests/test_e2e.py::TestInstanceLifecycle::test_dns_zone_in_create_output PASSED [ 66%]
tests/test_e2e.py::TestInstanceLifecycle::test_type_tag_applied PASSED [ 83%]
tests/test_e2e.py::TestInstanceLifecycle::test_custom_tag_applied PASSED [100%]
=================================== FAILURES ===================================
________________ TestInstanceLifecycle.test_dns_record_created _________________
self = <tests.test_e2e.TestInstanceLifecycle object at 0x70b9055aeab0>
instance = {'create_out': 'New droplet created with id: 67e69710-1f35-415f-819e-3e5d63e05ae3\nWARNING: Vultr support is experimen...sting.fyi\n', 'custom_tag': 'e2e-tag-a582f0', 'id': '67e69710-1f35-415f-819e-3e5d63e05ae3', 'name': 'e2etest-e9fe3877'}
config_file = PosixPath('/tmp/pytest-of-david/pytest-0/e2e0/config.yml')
session_id = 'cc08a013'
def test_dns_record_created(self, instance, config_file, session_id):
"""Verify that a DNS A record was created for the instance."""
result = run_machine(
"list-domain",
"--name",
instance["name"],
"--output",
"json",
E2E_DNS_ZONE,
config_file=config_file,
session_id=session_id,
)
assert result.returncode == 0, f"list-domain failed: {result.stderr}"
records = json.loads(result.stdout)
a_records = [r for r in records if r.get("name") == instance["name"] and r.get("type") == "A"]
> assert len(a_records) >= 1, f"No A record found for {instance['name']}.{E2E_DNS_ZONE}"
E AssertionError: No A record found for e2etest-e9fe3877.vultr-testing.fyi
E assert 0 >= 1
E + where 0 = len([])
tests/test_e2e.py:308: AssertionError
=========================== short test summary info ============================
FAILED tests/test_e2e.py::TestInstanceLifecycle::test_dns_record_created - As...
========================= 1 failed, 5 passed in 12.04s =========================Good - the DNS test fails. Let me check what happened with the DNS record and verify the 0.0.0.0 hypothesis. Let me also check if the VM got cleaned up.
E2E_PROVIDER=vultr uv run machine --config-file /tmp/pytest-of-david/pytest-0/e2e0/config.yml list --all --output json 2>/dev/null
vultr:
api-key: TK7MLDJBNLQVHIRMRYWFNWTVH5NOXSQ65IJQ
ssh-key: machine-testing
dns-zone: vultr-testing.fyi
machine-size: vc2-1c-1gb
image: 2284
region: ewr
machines:
e2e-basic:
new-user-name: e2euser[{"id": "fec4a831-69f8-4391-934a-f94762aa5a83", "name": "e2etest-585b0711", "tags": ["e2e-tag-e439bf", "machine:created", "machine:session:a3f07a43", "machine:type:e2e-basic"], "region": "ewr", "ip": "64.176.208.51", "type": "e2e-basic"}, {"id": "ba683fd0-7d89-4e13-bfca-21b69017ca9d", "name": "e2etest-6513028d", "tags": ["e2e-tag-4ec1cf", "machine:created", "machine:session:17580c4b", "machine:type:e2e-basic"], "region": "ewr", "ip": "64.176.221.202", "type": "e2e-basic"}, {"id": "c46d7c96-1690-4cae-8cfc-59a8c92c26c4", "name": "e2etest-bca5e72c", "tags": ["e2e-tag-01fbfa", "machine:created", "machine:session:9f3bee60", "machine:type:e2e-basic"], "region": "ewr", "ip": "149.28.44.122", "type": "e2e-basic"}, {"id": "139335ed-008f-4820-9236-2f400bc989f2", "name": "e2etest-988cb60b", "tags": ["e2e-tag-669a08", "machine:created", "machine:session:5276e98e", "machine:type:e2e-basic"], "region": "ewr", "ip": "45.32.1.204", "type": "e2e-basic"}, {"id": "67e69710-1f35-415f-819e-3e5d63e05ae3", "name": "e2etest-e9fe3877", "tags": ["e2e-tag-a582f0", "machine:created", "machine:session:cc08a013", "machine:type:e2e-basic"], "region": "ewr", "ip": "140.82.46.126", "type": "e2e-basic"}]This confirms both issues clearly:
Issue #57 confirmed: There are 5 leftover VMs from previous test runs that were never cleaned up. The most recent one (e2etest-e9fe3877) is from the test we just ran.
Issue #55 likely confirmed: DNS test fails. Let me check the DNS records to verify the 0.0.0.0 issue.
E2E_PROVIDER=vultr uv run machine --config-file /tmp/pytest-of-david/pytest-0/e2e0/config.yml list-domain --output json vultr-testing.fyi 2>/dev/null
E2E_PROVIDER=vultr uv run machine --config-file /tmp/pytest-of-david/pytest-0/e2e0/config.yml --verbose list-domain --all --output json vultr-testing.fyi 2>&1
WARNING: Vultr support is experimental and has not been fully verified. Use with caution.
[{"id": "901e5884-3b48-44b0-a876-abc76fc40c82", "droplet": null, "name": "", "fqdn": ".vultr-testing.fyi", "zone": "vultr-testing.fyi", "data": "45.76.31.247", "ttl": 300, "type": "A"}, {"id": "41ecbc06-08c0-40c3-828d-69ef4d9aa577", "droplet": null, "name": "e2etest-585b0711", "fqdn": "e2etest-585b0711.vultr-testing.fyi", "zone": "vultr-testing.fyi", "data": "0.0.0.0", "ttl": 300, "type": "A"}, {"id": "44b78141-2aa0-4f2f-9b13-16a1bfcb1530", "droplet": null, "name": "e2etest-6513028d", "fqdn": "e2etest-6513028d.vultr-testing.fyi", "zone": "vultr-testing.fyi", "data": "0.0.0.0", "ttl": 300, "type": "A"}, {"id": "7f9e122f-71a0-4bb8-94b0-893ab98c138f", "droplet": null, "name": "e2etest-bca5e72c", "fqdn": "e2etest-bca5e72c.vultr-testing.fyi", "zone": "vultr-testing.fyi", "data": "0.0.0.0", "ttl": 300, "type": "A"}, {"id": "52d29b1a-6fb7-4557-900d-b8f27ea33473", "droplet": null, "name": "e2etest-988cb60b", "fqdn": "e2etest-988cb60b.vultr-testing.fyi", "zone": "vultr-testing.fyi", "data": "0.0.0.0", "ttl": 300, "type": "A"}, {"id": "72e7f6e0-8b5e-4952-a278-a8de5a9d6fed", "droplet": null, "name": "e2etest-e9fe3877", "fqdn": "e2etest-e9fe3877.vultr-testing.fyi", "zone": "vultr-testing.fyi", "data": "0.0.0.0", "ttl": 300, "type": "A"}]All DNS records have 0.0.0.0 - confirming my diagnosis. Now let me also check why the list-domain without --all returned empty (to understand why the test failed). Let me look at the list-domain filtering.
{
"file_path": "/home/david/projects/bpi/machine/machine/subcommands/list_domain.py"
} 1→import click
2→import json
3→
4→from machine.log import fatal_error, output
5→from machine.types import MainCmdCtx, TAG_MACHINE_SESSION_PREFIX
6→from machine.util import dns_record_to_json_obj
7→
8→
9→def print_normal(records, zone):
10→ for record in records:
11→ output(f"{record.name}\t{record.type}\t{record.data}")
12→
13→
14→def print_quiet(records):
15→ for record in records:
16→ output(record.name)
17→
18→
19→def print_json(records, vms, zone):
20→ simplified = []
21→ for r in records:
22→ vm = next((v for v in vms if r.data == v.ip_address), None)
23→ simplified.append(dns_record_to_json_obj(r, zone, vm))
24→ output(json.dumps(simplified))
25→
26→
27→@click.command(help="List domain records")
28→@click.option("--name", "-n", metavar="<RECORD-NAME>", help="Filter by name")
29→@click.option("--type", "-m", metavar="<RECORD-TYPE>", help="Filter by type (default A and AAAA)")
30→@click.option("--output", "-o", metavar="<FORMAT>", help="Output format")
31→@click.option("--quiet", "-q", is_flag=True, default=False, help="Only display machine IDs")
32→@click.option(
33→ "--all",
34→ is_flag=True,
35→ default=False,
36→ help="Include all records, even those not created by this tool or created by other sessions",
37→)
38→@click.argument("zone", required=False)
39→@click.pass_context
40→def command(context, name, type, output, quiet, all, zone):
41→ command_context: MainCmdCtx = context.obj
42→ provider = command_context.provider
43→
44→ if not zone:
45→ zone = command_context.config.dns_zone
46→ if not zone:
47→ fatal_error("Error: no DNS zone specified.")
48→
49→ records = provider.get_dns_records(zone)
50→
51→ if type:
52→ if type != "*":
53→ records = filter(lambda r: r.type == type, records)
54→ else:
55→ records = filter(lambda r: r.type in ["A", "AAAA"], records)
56→
57→ if all:
58→ vms = provider.list_vms()
59→ else:
60→ vms = provider.list_vms(tag=TAG_MACHINE_SESSION_PREFIX + command_context.session_id)
61→ vm_ips = [v.ip_address for v in vms]
62→ records = filter(lambda r: r.data in vm_ips, records)
63→
64→ records = list(records)
65→ if output == "json":
66→ print_json(records, vms, zone)
67→ elif quiet:
68→ print_quiet(records)
69→ else:
70→ print_normal(records, zone)
71→Now I have a clear picture. Let me verify the cleanup issue by trying to manually destroy the VM from our test run.
E2E_PROVIDER=vultr uv run machine --config-file /tmp/pytest-of-david/pytest-0/e2e0/config.yml --session-id cc08a013 --verbose destroy --no-confirm --delete-dns 67e69710-1f35-415f-819e-3e5d63e05ae3 2>&1
E2E_PROVIDER=vultr uv run machine --config-file /tmp/pytest-of-david/pytest-0/e2e0/config.yml list --all --output json 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(len(d), 'VMs remaining'); [print(f' {x[\"name\"]} ({x[\"id\"]})') for x in d]"4 VMs remaining e2etest-585b0711 (fec4a831-69f8-4391-934a-f94762aa5a83) e2etest-6513028d (ba683fd0-7d89-4e13-bfca-21b69017ca9d) e2etest-bca5e72c (c46d7c96-1690-4cae-8cfc-59a8c92c26c4) e2etest-988cb60b (139335ed-008f-4820-9236-2f400bc989f2)
OK, so the manual destroy works fine. The question is why the teardown in tests doesn't work. The destroy command succeeded (no error output), and it did destroy the VM. So maybe the teardown IS running but only for the most recent test - the older 4 VMs are from previous CI runs where perhaps the teardown failed or wasn't reached.
Let me check if maybe the issue is that during CI, the test itself fails, causing a different code path during teardown. Actually, looking at the fixture more carefully - the fixture uses yield, so teardown should still run even on test failure. The destroy did work for our run. So the 4 remaining VMs are from previous CI runs.
Let me clean those up and then focus on the root cause analysis. Let me also check what the actual destroy output looks like with debug to understand if the DNS deletion part of destroy works.
E2E_PROVIDER=vultr uv run machine --config-file /tmp/pytest-of-david/pytest-0/e2e0/config.yml --session-id dummy --debug destroy --no-confirm --delete-dns --all fec4a831-69f8-4391-934a-f94762aa5a83 ba683fd0-7d89-4e13-bfca-21b69017ca9d c46d7c96-1690-4cae-8cfc-59a8c92c26c4 139335ed-008f-4820-9236-2f400bc989f2 2>&1
Loaded config file: /tmp/pytest-of-david/pytest-0/e2e0/config.yml
Parsed config: {'vultr': {'api-key': 'TK7MLDJBNLQVHIRMRYWFNWTVH5NOXSQ65IJQ', 'ssh-key': 'machine-testing', 'dns-zone': 'vultr-testing.fyi', 'machine-size': 'vc2-1c-1gb', 'image': 2284, 'region': 'ewr'}, 'machines': {'e2e-basic': {'new-user-name': 'e2euser'}}}
WARNING: Vultr support is experimental and has not been fully verified. Use with caution.
Deleting host record e2etest-585b0711.vultr-testing.fyi
Deleted dns record for e2etest-585b0711.vultr-testing.fyi
Deleting host record e2etest-6513028d.vultr-testing.fyi
Deleted dns record for e2etest-6513028d.vultr-testing.fyi
Deleting host record e2etest-bca5e72c.vultr-testing.fyi
Deleted dns record for e2etest-bca5e72c.vultr-testing.fyi
Deleting host record e2etest-988cb60b.vultr-testing.fyi
Deleted dns record for e2etest-988cb60b.vultr-testing.fyiGood, cleanup done. Now let me understand why the teardown fails in CI. Let me re-read the fixture teardown and the CI failure log.
gh api repos/stirlingbridge/machine/actions/runs/23165517312/jobs --jq '.jobs[] | select(.name | contains("vultr")) | .id' 2>&1gh api repos/stirlingbridge/machine/actions/jobs/67304330705/logs 2>&1 | head -200
2026-03-16T20:55:51.3354116Z Current runner version: '2.332.0'
2026-03-16T20:55:51.3378098Z ##[group]Runner Image Provisioner
2026-03-16T20:55:51.3379035Z Hosted Compute Agent
2026-03-16T20:55:51.3379573Z Version: 20260213.493
2026-03-16T20:55:51.3380172Z Commit: 5c115507f6dd24b8de37d8bbe0bb4509d0cc0fa3
2026-03-16T20:55:51.3380993Z Build Date: 2026-02-13T00:28:41Z
2026-03-16T20:55:51.3381631Z Worker ID: {8cdaba5f-b29e-470d-a2c3-3fbfb7883d4c}
2026-03-16T20:55:51.3382304Z Azure Region: westus
2026-03-16T20:55:51.3382908Z ##[endgroup]
2026-03-16T20:55:51.3384271Z ##[group]Operating System
2026-03-16T20:55:51.3384867Z Ubuntu
2026-03-16T20:55:51.3385503Z 24.04.3
2026-03-16T20:55:51.3385938Z LTS
2026-03-16T20:55:51.3386388Z ##[endgroup]
2026-03-16T20:55:51.3387164Z ##[group]Runner Image
2026-03-16T20:55:51.3387727Z Image: ubuntu-24.04
2026-03-16T20:55:51.3388192Z Version: 20260309.50.1
2026-03-16T20:55:51.3389488Z Included Software: https://github.com/actions/runner-images/blob/ubuntu24/20260309.50/images/ubuntu/Ubuntu2404-Readme.md
2026-03-16T20:55:51.3390948Z Image Release: https://github.com/actions/runner-images/releases/tag/ubuntu24%2F20260309.50
2026-03-16T20:55:51.3391889Z ##[endgroup]
2026-03-16T20:55:51.3394582Z ##[group]GITHUB_TOKEN Permissions
2026-03-16T20:55:51.3396876Z Actions: write
2026-03-16T20:55:51.3397486Z ArtifactMetadata: write
2026-03-16T20:55:51.3398140Z Attestations: write
2026-03-16T20:55:51.3398666Z Checks: write
2026-03-16T20:55:51.3399212Z Contents: write
2026-03-16T20:55:51.3400225Z Deployments: write
2026-03-16T20:55:51.3400790Z Discussions: write
2026-03-16T20:55:51.3401325Z Issues: write
2026-03-16T20:55:51.3401858Z Metadata: read
2026-03-16T20:55:51.3402345Z Models: read
2026-03-16T20:55:51.3402870Z Packages: write
2026-03-16T20:55:51.3403412Z Pages: write
2026-03-16T20:55:51.3403997Z PullRequests: write
2026-03-16T20:55:51.3404554Z RepositoryProjects: write
2026-03-16T20:55:51.3405176Z SecurityEvents: write
2026-03-16T20:55:51.3405735Z Statuses: write
2026-03-16T20:55:51.3406251Z ##[endgroup]
2026-03-16T20:55:51.3409037Z Secret source: Actions
2026-03-16T20:55:51.3409818Z Prepare workflow directory
2026-03-16T20:55:51.3735957Z Prepare all required actions
2026-03-16T20:55:51.3779056Z Getting action download info
2026-03-16T20:55:51.8638906Z Download action repository 'actions/checkout@v4' (SHA:34e114876b0b11c390a56381ad16ebd13914f8d5)
2026-03-16T20:55:52.0188289Z Download action repository 'astral-sh/setup-uv@v4' (SHA:38f3f104447c67c051c4a08e39b64a148898af3a)
2026-03-16T20:55:52.7456481Z Complete job name: E2E Tests (vultr)
2026-03-16T20:55:52.8219637Z ##[group]Run actions/checkout@v4
2026-03-16T20:55:52.8220569Z with:
2026-03-16T20:55:52.8221042Z repository: stirlingbridge/machine
2026-03-16T20:55:52.8221823Z token: ***
2026-03-16T20:55:52.8222247Z ssh-strict: true
2026-03-16T20:55:52.8222694Z ssh-user: git
2026-03-16T20:55:52.8223143Z persist-credentials: true
2026-03-16T20:55:52.8223654Z clean: true
2026-03-16T20:55:52.8224102Z sparse-checkout-cone-mode: true
2026-03-16T20:55:52.8224621Z fetch-depth: 1
2026-03-16T20:55:52.8225078Z fetch-tags: false
2026-03-16T20:55:52.8225533Z show-progress: true
2026-03-16T20:55:52.8225994Z lfs: false
2026-03-16T20:55:52.8226423Z submodules: false
2026-03-16T20:55:52.8227075Z set-safe-directory: true
2026-03-16T20:55:52.8227905Z ##[endgroup]
2026-03-16T20:55:52.9345702Z Syncing repository: stirlingbridge/machine
2026-03-16T20:55:52.9349216Z ##[group]Getting Git version info
2026-03-16T20:55:52.9350560Z Working directory is '/home/runner/work/machine/machine'
2026-03-16T20:55:52.9352626Z [command]/usr/bin/git version
2026-03-16T20:55:52.9431392Z git version 2.53.0
2026-03-16T20:55:52.9460210Z ##[endgroup]
2026-03-16T20:55:52.9486070Z Temporarily overriding HOME='/home/runner/work/_temp/5de8f5bc-83a4-4c40-8c37-0c8f39665382' before making global git config changes
2026-03-16T20:55:52.9488983Z Adding repository directory to the temporary git global config as a safe directory
2026-03-16T20:55:52.9493226Z [command]/usr/bin/git config --global --add safe.directory /home/runner/work/machine/machine
2026-03-16T20:55:52.9538363Z Deleting the contents of '/home/runner/work/machine/machine'
2026-03-16T20:55:52.9542783Z ##[group]Initializing the repository
2026-03-16T20:55:52.9548036Z [command]/usr/bin/git init /home/runner/work/machine/machine
2026-03-16T20:55:52.9685156Z hint: Using 'master' as the name for the initial branch. This default branch name
2026-03-16T20:55:52.9686606Z hint: will change to "main" in Git 3.0. To configure the initial branch name
2026-03-16T20:55:52.9688299Z hint: to use in all of your new repositories, which will suppress this warning,
2026-03-16T20:55:52.9689850Z hint: call:
2026-03-16T20:55:52.9690414Z hint:
2026-03-16T20:55:52.9691360Z hint: git config --global init.defaultBranch <name>
2026-03-16T20:55:52.9692695Z hint:
2026-03-16T20:55:52.9693946Z hint: Names commonly chosen instead of 'master' are 'main', 'trunk' and
2026-03-16T20:55:52.9695855Z hint: 'development'. The just-created branch can be renamed via this command:
2026-03-16T20:55:52.9697579Z hint:
2026-03-16T20:55:52.9698395Z hint: git branch -m <name>
2026-03-16T20:55:52.9699322Z hint:
2026-03-16T20:55:52.9700532Z hint: Disable this message with "git config set advice.defaultBranchName false"
2026-03-16T20:55:52.9702458Z Initialized empty Git repository in /home/runner/work/machine/machine/.git/
2026-03-16T20:55:52.9706672Z [command]/usr/bin/git remote add origin https://github.com/stirlingbridge/machine
2026-03-16T20:55:52.9748983Z ##[endgroup]
2026-03-16T20:55:52.9750375Z ##[group]Disabling automatic garbage collection
2026-03-16T20:55:52.9754239Z [command]/usr/bin/git config --local gc.auto 0
2026-03-16T20:55:52.9786363Z ##[endgroup]
2026-03-16T20:55:52.9788009Z ##[group]Setting up auth
2026-03-16T20:55:52.9794894Z [command]/usr/bin/git config --local --name-only --get-regexp core\.sshCommand
2026-03-16T20:55:52.9828158Z [command]/usr/bin/git submodule foreach --recursive sh -c "git config --local --name-only --get-regexp 'core\.sshCommand' && git config --local --unset-all 'core.sshCommand' || :"
2026-03-16T20:55:53.0201702Z [command]/usr/bin/git config --local --name-only --get-regexp http\.https\:\/\/github\.com\/\.extraheader
2026-03-16T20:55:53.0234342Z [command]/usr/bin/git submodule foreach --recursive sh -c "git config --local --name-only --get-regexp 'http\.https\:\/\/github\.com\/\.extraheader' && git config --local --unset-all 'http.https://github.com/.extraheader' || :"
2026-03-16T20:55:53.0469467Z [command]/usr/bin/git config --local --name-only --get-regexp ^includeIf\.gitdir:
2026-03-16T20:55:53.0503100Z [command]/usr/bin/git submodule foreach --recursive git config --local --show-origin --name-only --get-regexp remote.origin.url
2026-03-16T20:55:53.0744489Z [command]/usr/bin/git config --local http.https://github.com/.extraheader AUTHORIZATION: basic ***
2026-03-16T20:55:53.0782578Z ##[endgroup]
2026-03-16T20:55:53.0783558Z ##[group]Fetching the repository
2026-03-16T20:55:53.0791863Z [command]/usr/bin/git -c protocol.version=2 fetch --no-tags --prune --no-recurse-submodules --depth=1 origin +b0382b65b9bc16d72657d1965feff6bb1b66fc13:refs/remotes/origin/main
2026-03-16T20:55:53.5260682Z From https://github.com/stirlingbridge/machine
2026-03-16T20:55:53.5261788Z * [new ref] b0382b65b9bc16d72657d1965feff6bb1b66fc13 -> origin/main
2026-03-16T20:55:53.5294863Z ##[endgroup]
2026-03-16T20:55:53.5295594Z ##[group]Determining the checkout info
2026-03-16T20:55:53.5297483Z ##[endgroup]
2026-03-16T20:55:53.5302723Z [command]/usr/bin/git sparse-checkout disable
2026-03-16T20:55:53.5344826Z [command]/usr/bin/git config --local --unset-all extensions.worktreeConfig
2026-03-16T20:55:53.5373555Z ##[group]Checking out the ref
2026-03-16T20:55:53.5377634Z [command]/usr/bin/git checkout --progress --force -B main refs/remotes/origin/main
2026-03-16T20:55:53.5460220Z Switched to a new branch 'main'
2026-03-16T20:55:53.5463703Z branch 'main' set up to track 'origin/main'.
2026-03-16T20:55:53.5470890Z ##[endgroup]
2026-03-16T20:55:53.5507655Z [command]/usr/bin/git log -1 --format=%H
2026-03-16T20:55:53.5531015Z b0382b65b9bc16d72657d1965feff6bb1b66fc13
2026-03-16T20:55:53.5761254Z ##[group]Run astral-sh/setup-uv@v4
2026-03-16T20:55:53.5761890Z with:
2026-03-16T20:55:53.5762354Z version: latest
2026-03-16T20:55:53.5763025Z github-token: ***
2026-03-16T20:55:53.5763519Z enable-cache: false
2026-03-16T20:55:53.5764069Z cache-dependency-glob: **/uv.lock
2026-03-16T20:55:53.5764647Z prune-cache: true
2026-03-16T20:55:53.5765170Z ignore-nothing-to-cache: false
2026-03-16T20:55:53.5765722Z ##[endgroup]
2026-03-16T20:55:54.0631840Z Downloading uv from "https://github.com/astral-sh/uv/releases/download/0.10.10/uv-x86_64-unknown-linux-gnu.tar.gz" ...
2026-03-16T20:55:54.4862545Z [command]/usr/bin/tar xz --warning=no-unknown-keyword --overwrite -C /home/runner/work/_temp/db7a95d8-80b4-4f4d-847e-f722383e0f35 -f /home/runner/work/_temp/61c43e6c-7611-4567-b807-37f9b151810b
2026-03-16T20:55:54.9574397Z Added /opt/hostedtoolcache/uv/0.10.10/x86_64 to the path
2026-03-16T20:55:54.9576266Z Added /home/runner/.local/bin to the path
2026-03-16T20:55:54.9601935Z Set UV_CACHE_DIR to /home/runner/work/_temp/setup-uv-cache
2026-03-16T20:55:54.9603767Z Successfully installed uv version 0.10.10
2026-03-16T20:55:54.9776328Z ##[group]Run uv python install 3.13
2026-03-16T20:55:54.9778047Z [36;1muv python install 3.13[0m
2026-03-16T20:55:54.9838191Z shell: /usr/bin/bash -e {0}
2026-03-16T20:55:54.9839361Z env:
2026-03-16T20:55:54.9840456Z UV_CACHE_DIR: /home/runner/work/_temp/setup-uv-cache
2026-03-16T20:55:54.9841900Z ##[endgroup]
2026-03-16T20:55:55.0621661Z Downloading cpython-3.13.12-linux-x86_64-gnu (download) (33.4MiB)
2026-03-16T20:55:55.9945724Z Downloaded cpython-3.13.12-linux-x86_64-gnu (download)
2026-03-16T20:55:56.0021077Z Installed Python 3.13.12 in 999ms
2026-03-16T20:55:56.0021980Z + cpython-3.13.12-linux-x86_64-gnu (python3.13)
2026-03-16T20:55:56.0086191Z ##[group]Run uv sync
2026-03-16T20:55:56.0086471Z [36;1muv sync[0m
2026-03-16T20:55:56.0136517Z shell: /usr/bin/bash -e {0}
2026-03-16T20:55:56.0136984Z env:
2026-03-16T20:55:56.0137273Z UV_CACHE_DIR: /home/runner/work/_temp/setup-uv-cache
2026-03-16T20:55:56.0137589Z ##[endgroup]
2026-03-16T20:55:56.2136456Z Using CPython 3.13.12
2026-03-16T20:55:56.2137540Z Creating virtual environment at: .venv
2026-03-16T20:55:56.2152545Z Resolved 29 packages in 0.69ms
2026-03-16T20:55:56.2157464Z Building machine @ file:///home/runner/work/machine/machine
2026-03-16T20:55:56.2243887Z Downloading pygments (1.2MiB)
2026-03-16T20:55:56.2273031Z Downloading ruff (10.8MiB)
2026-03-16T20:55:56.3670597Z Downloaded ruff
2026-03-16T20:55:56.4210475Z Downloaded pygments
2026-03-16T20:55:56.9247298Z Built machine @ file:///home/runner/work/machine/machine
2026-03-16T20:55:56.9259289Z Prepared 18 packages in 717ms
2026-03-16T20:55:56.9399416Z Installed 18 packages in 13ms
2026-03-16T20:55:56.9400281Z + certifi==2026.2.25
2026-03-16T20:55:56.9400871Z + charset-normalizer==3.4.5
2026-03-16T20:55:56.9401684Z + click==8.1.7
2026-03-16T20:55:56.9402153Z + expandvars==1.0.0
2026-03-16T20:55:56.9402515Z + idna==3.11
2026-03-16T20:55:56.9402846Z + iniconfig==2.3.0
2026-03-16T20:55:56.9403875Z + jsonpickle==4.1.1
2026-03-16T20:55:56.9404380Z + machine==1.0.0 (from file:///home/runner/work/machine/machine)
2026-03-16T20:55:56.9404985Z + packaging==26.0
2026-03-16T20:55:56.9405380Z + pluggy==1.6.0
2026-03-16T20:55:56.9405757Z + pygments==2.19.2
2026-03-16T20:55:56.9406100Z + pytest==9.0.2
2026-03-16T20:55:56.9406477Z + python-digitalocean==1.17.0
2026-03-16T20:55:56.9407115Z + requests==2.32.3
2026-03-16T20:55:56.9409438Z + ruamel-yaml==0.17.32
2026-03-16T20:55:56.9410542Z + ruff==0.15.6
2026-03-16T20:55:56.9411315Z + urllib3==2.6.3
2026-03-16T20:55:56.9412173Z + vultr-python==0.2.0
2026-03-16T20:55:56.9544332Z ##[group]Run uv run pytest tests/test_e2e.py -v -m e2e
2026-03-16T20:55:56.9544765Z [36;1muv run pytest tests/test_e2e.py -v -m e2e[0m
2026-03-16T20:55:56.9593718Z shell: /usr/bin/bash -e {0}
2026-03-16T20:55:56.9593962Z env:
2026-03-16T20:55:56.9594428Z UV_CACHE_DIR: /home/runner/work/_temp/setup-uv-cache
2026-03-16T20:55:56.9594737Z E2E_PROVIDER: vultr
2026-03-16T20:55:56.9594945Z E2E_SSH_KEY: machine-testing
2026-03-16T20:55:56.9595590Z E2E_DO_TOKEN: ***
2026-03-16T20:55:56.9595833Z E2E_DO_DNS_ZONE: machine-testing.servesthe.world
2026-03-16T20:55:56.9596127Z E2E_PROJECT: Testing
2026-03-16T20:55:56.9596411Z E2E_VULTR_API_KEY: ***
2026-03-16T20:55:56.9596650Z E2E_VULTR_DNS_ZONE: vultr-testing.fyi
2026-03-16T20:55:56.9597055Z ##[endgroup]
2026-03-16T20:55:57.4623288Z ============================= test session starts ==============================
2026-03-16T20:55:57.4624292Z platform linux -- Python 3.13.12, pytest-9.0.2, pluggy-1.6.0 -- /home/runner/work/machine/machine/.venv/bin/python
2026-03-16T20:55:57.4625051Z cachedir: .pytest_cache
2026-03-16T20:55:57.4625394Z rootdir: /home/runner/work/machine/machine
2026-03-16T20:55:57.4625802Z configfile: pyproject.toml
2026-03-16T20:55:57.4792734Z collecting ... collected 6 items
2026-03-16T20:55:57.4793197Z
2026-03-16T20:56:02.2412189Z tests/test_e2e.py::TestInstanceLifecycle::test_instance_appears_in_list PASSED [ 16%]
2026-03-16T20:56:03.2642045Z tests/test_e2e.py::TestInstanceLifecycle::test_instance_has_ip PASSED [ 33%]
2026-03-16T20:56:04.4008742Z tests/test_e2e.py::TestInstanceLifecycle::test_dns_record_created FAILED [ 50%]gh api repos/stirlingbridge/machine/actions/jobs/67304330705/logs 2>&1 | tail -60
2026-03-16T20:55:57.4792734Z collecting ... collected 6 items
2026-03-16T20:55:57.4793197Z
2026-03-16T20:56:02.2412189Z tests/test_e2e.py::TestInstanceLifecycle::test_instance_appears_in_list PASSED [ 16%]
2026-03-16T20:56:03.2642045Z tests/test_e2e.py::TestInstanceLifecycle::test_instance_has_ip PASSED [ 33%]
2026-03-16T20:56:04.4008742Z tests/test_e2e.py::TestInstanceLifecycle::test_dns_record_created FAILED [ 50%]
2026-03-16T20:56:04.4014720Z tests/test_e2e.py::TestInstanceLifecycle::test_dns_zone_in_create_output PASSED [ 66%]
2026-03-16T20:56:04.9744705Z tests/test_e2e.py::TestInstanceLifecycle::test_type_tag_applied PASSED [ 83%]
2026-03-16T20:56:06.7533048Z tests/test_e2e.py::TestInstanceLifecycle::test_custom_tag_applied PASSED [100%]
2026-03-16T20:56:06.7533789Z
2026-03-16T20:56:06.7534097Z =================================== FAILURES ===================================
2026-03-16T20:56:06.7535168Z ________________ TestInstanceLifecycle.test_dns_record_created _________________
2026-03-16T20:56:06.7535998Z
2026-03-16T20:56:06.7536513Z self = <tests.test_e2e.TestInstanceLifecycle object at 0x7fee897c9a70>
2026-03-16T20:56:06.7539733Z instance = {'create_out': 'New droplet created with id: 139335ed-008f-4820-9236-2f400bc989f2\nWARNING: Vultr support is experimen...sting.fyi\n', 'custom_tag': 'e2e-tag-669a08', 'id': '139335ed-008f-4820-9236-2f400bc989f2', 'name': 'e2etest-988cb60b'}
2026-03-16T20:56:06.7542747Z config_file = PosixPath('/tmp/pytest-of-runner/pytest-0/e2e0/config.yml')
2026-03-16T20:56:06.7543582Z session_id = '5276e98e'
2026-03-16T20:56:06.7543910Z
2026-03-16T20:56:06.7544246Z def test_dns_record_created(self, instance, config_file, session_id):
2026-03-16T20:56:06.7545053Z """Verify that a DNS A record was created for the instance."""
2026-03-16T20:56:06.7545772Z result = run_machine(
2026-03-16T20:56:06.7546224Z "list-domain",
2026-03-16T20:56:06.7546659Z "--name",
2026-03-16T20:56:06.7547291Z instance["name"],
2026-03-16T20:56:06.7547786Z "--output",
2026-03-16T20:56:06.7548321Z "json",
2026-03-16T20:56:06.7548830Z E2E_DNS_ZONE,
2026-03-16T20:56:06.7549347Z config_file=config_file,
2026-03-16T20:56:06.7549946Z session_id=session_id,
2026-03-16T20:56:06.7550475Z )
2026-03-16T20:56:06.7551162Z assert result.returncode == 0, f"list-domain failed: {result.stderr}"
2026-03-16T20:56:06.7552023Z records = json.loads(result.stdout)
2026-03-16T20:56:06.7552853Z a_records = [r for r in records if r.get("name") == instance["name"] and r.get("type") == "A"]
2026-03-16T20:56:06.7553834Z > assert len(a_records) >= 1, f"No A record found for {instance['name']}.{E2E_DNS_ZONE}"
2026-03-16T20:56:06.7555134Z E AssertionError: No A record found for e2etest-988cb60b.vultr-testing.fyi
2026-03-16T20:56:06.7555863Z E assert 0 >= 1
2026-03-16T20:56:06.7556255Z E + where 0 = len([])
2026-03-16T20:56:06.7556521Z
2026-03-16T20:56:06.7556691Z tests/test_e2e.py:308: AssertionError
2026-03-16T20:56:06.7557764Z =========================== short test summary info ============================
2026-03-16T20:56:06.7559075Z FAILED tests/test_e2e.py::TestInstanceLifecycle::test_dns_record_created - AssertionError: No A record found for e2etest-988cb60b.vultr-testing.fyi
2026-03-16T20:56:06.7560217Z assert 0 >= 1
2026-03-16T20:56:06.7560530Z + where 0 = len([])
2026-03-16T20:56:06.7560989Z ========================= 1 failed, 5 passed in 9.27s ==========================
2026-03-16T20:56:06.7768143Z ##[error]Process completed with exit code 1.
2026-03-16T20:56:06.7877933Z Post job cleanup.
2026-03-16T20:56:06.8825191Z [command]/usr/bin/git version
2026-03-16T20:56:06.8863684Z git version 2.53.0
2026-03-16T20:56:06.8908678Z Temporarily overriding HOME='/home/runner/work/_temp/4a74d939-27ed-434a-898b-b7b9120c6ccd' before making global git config changes
2026-03-16T20:56:06.8910034Z Adding repository directory to the temporary git global config as a safe directory
2026-03-16T20:56:06.8915254Z [command]/usr/bin/git config --global --add safe.directory /home/runner/work/machine/machine
2026-03-16T20:56:06.8958883Z [command]/usr/bin/git config --local --name-only --get-regexp core\.sshCommand
2026-03-16T20:56:06.8993090Z [command]/usr/bin/git submodule foreach --recursive sh -c "git config --local --name-only --get-regexp 'core\.sshCommand' && git config --local --unset-all 'core.sshCommand' || :"
2026-03-16T20:56:06.9234286Z [command]/usr/bin/git config --local --name-only --get-regexp http\.https\:\/\/github\.com\/\.extraheader
2026-03-16T20:56:06.9256465Z http.https://github.com/.extraheader
2026-03-16T20:56:06.9269427Z [command]/usr/bin/git config --local --unset-all http.https://github.com/.extraheader
2026-03-16T20:56:06.9301604Z [command]/usr/bin/git submodule foreach --recursive sh -c "git config --local --name-only --get-regexp 'http\.https\:\/\/github\.com\/\.extraheader' && git config --local --unset-all 'http.https://github.com/.extraheader' || :"
2026-03-16T20:56:06.9533369Z [command]/usr/bin/git config --local --name-only --get-regexp ^includeIf\.gitdir:
2026-03-16T20:56:06.9566242Z [command]/usr/bin/git submodule foreach --recursive git config --local --show-origin --name-only --get-regexp remote.origin.url
2026-03-16T20:56:06.9912383Z Cleaning up orphan processes
2026-03-16T20:56:07.0182797Z ##[warning]Node.js 20 actions are deprecated. The following actions are running on Node.js 20 and may not work as expected: actions/checkout@v4, astral-sh/setup-uv@v4. Actions will be forced to run with Node.js 24 by default starting June 2nd, 2026. Please check if updated versions of these actions are available that support Node.js 24. To opt into Node.js 24 now, set the FORCE_JAVASCRIPT_ACTIONS_TO_NODE24=true environment variable on the runner or in your workflow file. Once Node.js 24 becomes the default, you can temporarily opt out by setting ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true. For more information see: https://github.blog/changelog/2025-09-19-deprecation-of-node-20-on-github-actions-runners/E2E_PROVIDER=vultr uv run python3 -c "
from vultr import Vultr
import os
client = Vultr(os.environ['E2E_VULTR_API_KEY'])
# Create a test instance to examine
instances = client.list_instances()
for i in instances:
print(f'ID: {i[\"id\"]}, Name: {i[\"label\"]}, IP: {i[\"main_ip\"]}, Tags: {i[\"tags\"]}')
" 2>&1Good - no instances left (we cleaned them up). Let me now create a test instance, observe the initial IP, and trace through the full create/destroy cycle to pinpoint the issues.
E2E_PROVIDER=vultr uv run python3 -c "
from vultr import Vultr
import os, time
client = Vultr(os.environ['E2E_VULTR_API_KEY'])
# Create instance
result = client.create_instance('ewr', 'vc2-1c-1gb', os_id=2284, label='debug-test', hostname='debug-test', tags=['machine:created', 'machine:session:debug1'])
print(f'Created: id={result[\"id\"]}, main_ip={result[\"main_ip\"]!r}, status={result[\"status\"]}')
# Check IP over time
for i in range(15):
time.sleep(2)
inst = client.get_instance(result['id'])
print(f' t={i*2}s: main_ip={inst[\"main_ip\"]!r}, status={inst[\"status\"]}')
if inst['main_ip'] and inst['main_ip'] != '0.0.0.0':
break
# Clean up
client.delete_instance(result['id'])
print('Destroyed')
" 2>&1Exit code 1
Traceback (most recent call last):
File "<string>", line 19, in <module>
File "/home/david/projects/bpi/machine/.venv/lib/python3.12/site-packages/vultr/vultr.py", line 103, in delete_instance
return self._req("delete", url)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/david/projects/bpi/machine/.venv/lib/python3.12/site-packages/vultr/vultr.py", line 215, in _req
raise VultrException(r)
vultr.vultr.VultrException: Error 500: Unable to destroy server: This subscription is not currently active, you cannot destroy it.
Created: id=fb223312-0c88-432d-bfcf-60b3ea2aeac0, main_ip='0.0.0.0', status=pending
t=0s: main_ip='0.0.0.0', status=pending
t=2s: main_ip='0.0.0.0', status=pending
t=4s: main_ip='0.0.0.0', status=pending
t=6s: main_ip='0.0.0.0', status=pending
t=8s: main_ip='0.0.0.0', status=pending
t=10s: main_ip='0.0.0.0', status=pending
t=12s: main_ip='0.0.0.0', status=pending
t=14s: main_ip='0.0.0.0', status=pending
t=16s: main_ip='0.0.0.0', status=pending
t=18s: main_ip='0.0.0.0', status=pending
t=20s: main_ip='0.0.0.0', status=pending
t=22s: main_ip='0.0.0.0', status=pending
t=24s: main_ip='0.0.0.0', status=pending
t=26s: main_ip='0.0.0.0', status=pending
t=28s: main_ip='0.0.0.0', status=pendingExcellent! This reveals both root causes:
IP is 0.0.0.0 for a long time (30+ seconds) while the instance is pending. The code treats "0.0.0.0" as a valid IP and skips the wait loop.
You can't destroy a pending instance on Vultr - it returns HTTP 500: "This subscription is not currently active, you cannot destroy it." This is why the teardown silently fails and VMs accumulate!
Let me wait for this instance to become active and clean it up, then fix both issues.
E2E_PROVIDER=vultr uv run python3 -c "
from vultr import Vultr
import os, time
client = Vultr(os.environ['E2E_VULTR_API_KEY'])
# Wait for it to become active
for i in range(30):
inst = client.get_instance('fb223312-0c88-432d-bfcf-60b3ea2aeac0')
print(f't={i*5}s: main_ip={inst[\"main_ip\"]!r}, status={inst[\"status\"]}')
if inst['status'] == 'active':
client.delete_instance(inst['id'])
print('Destroyed successfully')
break
time.sleep(5)
" 2>&1Now I have a complete diagnosis. Here's the summary of all issues:
"0.0.0.0" treated as valid IP (Issues #55, #53, #61)In machine/subcommands/create.py:87:
ip_address = vm.ip_address # Returns "0.0.0.0" on Vultr
if (wait_for_ip or update_dns) and not ip_address: # "0.0.0.0" is truthy → skips wait!
Vultr newly-created instances return main_ip: "0.0.0.0" while status: "pending". Since "0.0.0.0" is a non-empty string, not ip_address is False, so the wait-for-IP loop is never entered. The DNS record then gets created with 0.0.0.0.
The list-domain command then can't match DNS records (data 0.0.0.0) to VMs (which eventually get real IPs), so the DNS test fails.
pending Vultr instances (Issue #57)Vultr returns HTTP 500 when you try to destroy an instance that is still in pending status: "This subscription is not currently active, you cannot destroy it."
Because the create command returns almost instantly (the 0.0.0.0 IP skips the wait), the tests complete in ~9 seconds. The teardown fixture then tries to destroy the VM while it's still pending, which fails. The destroy_vm call in vultr.py catches the VultrException via fatal_error() which exits the subprocess, but the test teardown ignores the subprocess return code.
create.py: Treat "0.0.0.0" as "no IP yet" in the wait loopvultr.py: In destroy_vm, wait for the instance to become active before attempting deletion (or retry)test_e2e.py: test_instance_has_ip should validate the IP is not 0.0.0.0Want me to implement these fixes?
# If requested, or if we are going to set a DNS record get the VM's IPv4 address
ip_address = vm.ip_address
if (wait_for_ip or update_dns) and not ip_address:
while not ip_address: # If requested, or if we are going to set a DNS record get the VM's IPv4 address
# Vultr returns "0.0.0.0" as main_ip while the instance is still pending,
# so treat that the same as no IP assigned yet.
ip_address = vm.ip_address if vm.ip_address != "0.0.0.0" else None
if (wait_for_ip or update_dns) and not ip_address:
while not ip_address:The file /home/david/projects/bpi/machine/machine/subcommands/create.py has been updated successfully.
{
"file_path": "/home/david/projects/bpi/machine/machine/subcommands/create.py",
"offset": 87,
"limit": 15
} 87→ # so treat that the same as no IP assigned yet.
88→ ip_address = vm.ip_address if vm.ip_address != "0.0.0.0" else None
89→ if (wait_for_ip or update_dns) and not ip_address:
90→ while not ip_address:
91→ time.sleep(1)
92→ vm = provider.get_vm(vm.id)
93→ ip_address = vm.ip_address
94→ if d.opt.verbose:
95→ output("Waiting for droplet IP address")
96→ if d.opt.quiet:
97→ info(f"{ip_address}")
98→ else:
99→ info(f"IP Address: {ip_address}")
100→
101→ # If requested, and we have the IP address, create a DNS host recordThe inner loop also needs the same fix - when polling, vm.ip_address could still be "0.0.0.0":
while not ip_address:
time.sleep(1)
vm = provider.get_vm(vm.id)
ip_address = vm.ip_address while not ip_address:
time.sleep(1)
vm = provider.get_vm(vm.id)
ip_address = vm.ip_address if vm.ip_address != "0.0.0.0" else NoneThe file /home/david/projects/bpi/machine/machine/subcommands/create.py has been updated successfully.
import base64 from vultr import Vultr, VultrException from machine.log import fatal_error, info from machine.provider import CloudProvider, VM, SSHKey, DNSRecord
import base64 import time from vultr import Vultr, VultrException from machine.log import fatal_error, info from machine.provider import CloudProvider, VM, SSHKey, DNSRecord
The file /home/david/projects/bpi/machine/machine/providers/vultr.py has been updated successfully.
def destroy_vm(self, vm_id) -> bool:
try:
self._client.delete_instance(vm_id)
except VultrException as e:
fatal_error(f"Error: machine with id {vm_id} not found: {e}")
return True def destroy_vm(self, vm_id) -> bool:
# Vultr returns HTTP 500 if the instance is still pending ("not currently active").
# Poll until the instance is active (or gone) before attempting deletion.
for _ in range(60):
try:
instance = self._client.get_instance(vm_id)
except VultrException:
return True # instance already gone
if instance.get("status") != "pending":
break
info("Waiting for instance to become active before destroying...")
time.sleep(5)
try:
self._client.delete_instance(vm_id)
except VultrException as e:
fatal_error(f"Error: machine with id {vm_id} not found: {e}")
return TrueThe file /home/david/projects/bpi/machine/machine/providers/vultr.py has been updated successfully.
assert matched[0]["ip"] is not None, "Instance has no IP address"
ip = matched[0]["ip"]
assert ip is not None, "Instance has no IP address"
assert ip != "0.0.0.0", "Instance IP is 0.0.0.0 (not yet assigned)"The file /home/david/projects/bpi/machine/tests/test_e2e.py has been updated successfully.
uv run ruff check machine/subcommands/create.py machine/providers/vultr.py tests/test_e2e.py 2>&1
E2E_PROVIDER=vultr uv run pytest tests/test_e2e.py -v -m e2e 2>&1
============================= test session starts ============================== platform linux -- Python 3.12.3, pytest-9.0.2, pluggy-1.6.0 -- /home/david/projects/bpi/machine/.venv/bin/python3 cachedir: .pytest_cache rootdir: /home/david/projects/bpi/machine configfile: pyproject.toml collecting ... collected 6 items tests/test_e2e.py::TestInstanceLifecycle::test_instance_appears_in_list PASSED [ 16%] tests/test_e2e.py::TestInstanceLifecycle::test_instance_has_ip PASSED [ 33%] tests/test_e2e.py::TestInstanceLifecycle::test_dns_record_created PASSED [ 50%] tests/test_e2e.py::TestInstanceLifecycle::test_dns_zone_in_create_output PASSED [ 66%] tests/test_e2e.py::TestInstanceLifecycle::test_type_tag_applied PASSED [ 83%] tests/test_e2e.py::TestInstanceLifecycle::test_custom_tag_applied PASSED [100%] ============================== 6 passed in 29.14s ==============================
E2E_PROVIDER=vultr uv run python3 -c "
from vultr import Vultr
import os
client = Vultr(os.environ['E2E_VULTR_API_KEY'])
instances = client.list_instances()
print(f'{len(instances)} instances remaining')
for i in instances:
print(f' {i[\"label\"]} ({i[\"id\"]})')
" 2>&1E2E_PROVIDER=vultr uv run python3 -c "
from vultr import Vultr
import os, time
client = Vultr(os.environ['E2E_VULTR_API_KEY'])
# Check status and wait if needed
inst = client.get_instance('6b82e4c1-2e58-4803-a4de-4d8b5cbfe15e')
print(f'Status: {inst[\"status\"]}, IP: {inst[\"main_ip\"]}')
if inst['status'] == 'pending':
print('Still pending, waiting...')
for _ in range(30):
time.sleep(5)
inst = client.get_instance('6b82e4c1-2e58-4803-a4de-4d8b5cbfe15e')
if inst['status'] != 'pending':
break
client.delete_instance('6b82e4c1-2e58-4803-a4de-4d8b5cbfe15e')
print('Cleaned up')
" 2>&1One VM survived the teardown. Let me re-run the test to confirm the cleanup works end-to-end, since the remaining VM may have been a race condition from the earlier test.
E2E_PROVIDER=vultr uv run pytest tests/test_e2e.py -v -m e2e -s 2>&1
============================= test session starts ============================== platform linux -- Python 3.12.3, pytest-9.0.2, pluggy-1.6.0 -- /home/david/projects/bpi/machine/.venv/bin/python3 cachedir: .pytest_cache rootdir: /home/david/projects/bpi/machine configfile: pyproject.toml collecting ... collected 6 items tests/test_e2e.py::TestInstanceLifecycle::test_instance_appears_in_list PASSED tests/test_e2e.py::TestInstanceLifecycle::test_instance_has_ip PASSED tests/test_e2e.py::TestInstanceLifecycle::test_dns_record_created PASSED tests/test_e2e.py::TestInstanceLifecycle::test_dns_zone_in_create_output PASSED tests/test_e2e.py::TestInstanceLifecycle::test_type_tag_applied PASSED tests/test_e2e.py::TestInstanceLifecycle::test_custom_tag_applied PASSED ============================== 6 passed in 21.49s ==============================
E2E_PROVIDER=vultr uv run python3 -c "
from vultr import Vultr; import os
client = Vultr(os.environ['E2E_VULTR_API_KEY'])
instances = client.list_instances()
print(f'{len(instances)} instances remaining')
" 2>&1E2E_PROVIDER=vultr uv run python3 -c "
from vultr import Vultr; import os
client = Vultr(os.environ['E2E_VULTR_API_KEY'])
for i in client.list_instances():
print(f'{i[\"label\"]} ({i[\"id\"]}) status={i[\"status\"]} tags={i[\"tags\"]}')
" 2>&1e2etest-d2e7f999 (94bff61e-20c6-4dd7-b23c-473c581f3251) status=active tags=['e2e-tag-eb4a4a', 'machine:created', 'machine:session:7565844f', 'machine:type:e2e-basic']
The instance has proper tags. Let me manually test the destroy command to see what happens:
E2E_PROVIDER=vultr uv run machine --config-file /tmp/pytest-of-david/pytest-0/e2e0/config.yml --session-id 7565844f --verbose destroy --no-confirm --delete-dns 94bff61e-20c6-4dd7-b23c-473c581f3251 2>&1; echo "EXIT: $?"
WARNING: Vultr support is experimental and has not been fully verified. Use with caution. EXIT: 0
That works. So the destroy command itself works when run directly. The issue must be in how the teardown runs it. Let me check if there's a timing issue with the pytest teardown.
{
"file_path": "/home/david/projects/bpi/machine/tests/test_e2e.py",
"offset": 199,
"limit": 50
} 199→@pytest.fixture(scope="class")
200→def instance(config_file, session_id):
201→ """Create a single instance with all features and destroy it after all tests.
202→
203→ The instance is created with DNS, a machine type (cloud-init), a custom tag,
204→ and --wait-for-ip so that all aspects can be verified by individual tests.
205→ """
206→ name = _unique_name()
207→ custom_tag = f"e2e-tag-{uuid.uuid4().hex[:6]}"
208→
209→ # ---- CREATE with all features ------------------------------------------
210→ result = run_machine(
211→ "create",
212→ "--name",
213→ name,
214→ "--type",
215→ "e2e-basic",
216→ "--update-dns",
217→ "--tag",
218→ custom_tag,
219→ "--wait-for-ip",
220→ config_file=config_file,
221→ session_id=session_id,
222→ )
223→ assert result.returncode == 0, f"create failed: {result.stderr}"
224→ create_out = result.stdout + result.stderr
225→ instance_id = _extract_instance_id(create_out)
226→ assert instance_id, f"Could not find instance id in output:\n{create_out}"
227→
228→ info = {
229→ "name": name,
230→ "id": instance_id,
231→ "custom_tag": custom_tag,
232→ "create_out": create_out,
233→ }
234→
235→ yield info
236→
237→ # ---- TEARDOWN: destroy with DNS cleanup --------------------------------
238→ run_machine(
239→ "--verbose",
240→ "destroy",
241→ "--no-confirm",
242→ "--delete-dns",
243→ instance_id,
244→ config_file=config_file,
245→ session_id=session_id,
246→ )
247→
248→ # ---- TEARDOWN: destroy with DNS cleanup --------------------------------
run_machine(
"--verbose",
"destroy",
"--no-confirm",
"--delete-dns",
instance_id,
config_file=config_file,
session_id=session_id,
) # ---- TEARDOWN: destroy with DNS cleanup --------------------------------
destroy_result = run_machine(
"--verbose",
"destroy",
"--no-confirm",
"--delete-dns",
instance_id,
config_file=config_file,
session_id=session_id,
)
if destroy_result.returncode != 0:
print(f"TEARDOWN WARNING: destroy exited {destroy_result.returncode}", flush=True)
print(f" stdout: {destroy_result.stdout}", flush=True)
print(f" stderr: {destroy_result.stderr}", flush=True)The file /home/david/projects/bpi/machine/tests/test_e2e.py has been updated successfully.
E2E_PROVIDER=vultr uv run pytest tests/test_e2e.py -v -m e2e -s 2>&1
============================= test session starts ============================== platform linux -- Python 3.12.3, pytest-9.0.2, pluggy-1.6.0 -- /home/david/projects/bpi/machine/.venv/bin/python3 cachedir: .pytest_cache rootdir: /home/david/projects/bpi/machine configfile: pyproject.toml collecting ... collected 6 items tests/test_e2e.py::TestInstanceLifecycle::test_instance_appears_in_list PASSED tests/test_e2e.py::TestInstanceLifecycle::test_instance_has_ip PASSED tests/test_e2e.py::TestInstanceLifecycle::test_dns_record_created PASSED tests/test_e2e.py::TestInstanceLifecycle::test_dns_zone_in_create_output PASSED tests/test_e2e.py::TestInstanceLifecycle::test_type_tag_applied PASSED tests/test_e2e.py::TestInstanceLifecycle::test_custom_tag_applied PASSEDTEARDOWN WARNING: destroy exited 1 stdout: stderr: WARNING: Vultr support is experimental and has not been fully verified. Use with caution. Error: machine with id 3e8e7c35-d5df-4265-b3f8-981a1ea058e7 not found: Error 500: Unable to destroy server: Unable to remove VM: Server is currently locked ============================== 6 passed in 29.18s ==============================
Now I can see the issue: "Server is currently locked". Vultr returns HTTP 500 when the server is still locked (being provisioned). The pending status check isn't sufficient - the server can be active but still locked. Let me update the destroy to also handle the locked state with a retry.
def destroy_vm(self, vm_id) -> bool:
# Vultr returns HTTP 500 if the instance is still pending ("not currently active").
# Poll until the instance is active (or gone) before attempting deletion.
for _ in range(60):
try:
instance = self._client.get_instance(vm_id)
except VultrException:
return True # instance already gone
if instance.get("status") != "pending":
break
info("Waiting for instance to become active before destroying...")
time.sleep(5)
try:
self._client.delete_instance(vm_id)
except VultrException as e:
fatal_error(f"Error: machine with id {vm_id} not found: {e}")
return True def destroy_vm(self, vm_id) -> bool:
# Vultr returns HTTP 500 if the instance is still pending or locked
# (e.g. during provisioning). Retry deletion with backoff.
for attempt in range(24):
try:
self._client.delete_instance(vm_id)
return True
except VultrException as e:
error_msg = str(e)
if "500" in error_msg and ("not currently active" in error_msg or "currently locked" in error_msg):
info("Waiting for instance to become ready before destroying...")
time.sleep(5)
elif "404" in error_msg:
return True # already gone
else:
fatal_error(f"Error: machine with id {vm_id} not found: {e}")
fatal_error(f"Error: timed out waiting to destroy instance {vm_id}")
return FalseThe file /home/david/projects/bpi/machine/machine/providers/vultr.py has been updated successfully.
E2E_PROVIDER=vultr uv run python3 -c "
from vultr import Vultr; import os
client = Vultr(os.environ['E2E_VULTR_API_KEY'])
for i in client.list_instances():
print(f'Deleting {i[\"label\"]} ({i[\"id\"]})')
client.delete_instance(i['id'])
print('Done')
" 2>&1E2E_PROVIDER=vultr uv run pytest tests/test_e2e.py -v -m e2e -s 2>&1
============================= test session starts ============================== platform linux -- Python 3.12.3, pytest-9.0.2, pluggy-1.6.0 -- /home/david/projects/bpi/machine/.venv/bin/python3 cachedir: .pytest_cache rootdir: /home/david/projects/bpi/machine configfile: pyproject.toml collecting ... collected 6 items tests/test_e2e.py::TestInstanceLifecycle::test_instance_appears_in_list PASSED tests/test_e2e.py::TestInstanceLifecycle::test_instance_has_ip PASSED tests/test_e2e.py::TestInstanceLifecycle::test_dns_record_created PASSED tests/test_e2e.py::TestInstanceLifecycle::test_dns_zone_in_create_output PASSED tests/test_e2e.py::TestInstanceLifecycle::test_type_tag_applied PASSED tests/test_e2e.py::TestInstanceLifecycle::test_custom_tag_applied PASSED ========================= 6 passed in 66.12s (0:01:06) =========================
E2E_PROVIDER=vultr uv run python3 -c "
from vultr import Vultr; import os
client = Vultr(os.environ['E2E_VULTR_API_KEY'])
instances = client.list_instances()
print(f'{len(instances)} instances remaining')
# Also check DNS records
result = client.get(f'/domains/vultr-testing.fyi/records')
records = [r for r in result.get('records', []) if r.get('type') == 'A' and r.get('name', '').startswith('e2etest')]
print(f'{len(records)} leftover e2etest DNS records')
" 2>&1VMs are fully cleaned up. There are 2 stale DNS records from earlier runs - let me clean those up too:
E2E_PROVIDER=vultr uv run python3 -c "
from vultr import Vultr; import os
client = Vultr(os.environ['E2E_VULTR_API_KEY'])
result = client.get('/domains/vultr-testing.fyi/records')
for r in result.get('records', []):
if r.get('type') == 'A' and r.get('name', '').startswith('e2etest'):
print(f'Deleting DNS record: {r[\"name\"]} -> {r[\"data\"]}')
client.delete(f'/domains/vultr-testing.fyi/records/{r[\"id\"]}')
print('Done')
" 2>&1Deleting DNS record: e2etest-fa4038a4 -> 66.135.30.122 Deleting DNS record: e2etest-e99d1812 -> 207.246.81.117 Done
============================= test session starts ============================== platform linux -- Python 3.12.3, pytest-9.0.2, pluggy-1.6.0 -- /home/david/projects/bpi/machine/.venv/bin/python3 cachedir: .pytest_cache rootdir: /home/david/projects/bpi/machine configfile: pyproject.toml collecting ... collected 29 items tests/test_cli_integration.py::TestVersionCommand::test_version_runs PASSED [ 3%] tests/test_cli_integration.py::TestVersionCommand::test_help_runs PASSED [ 6%] tests/test_cli_integration.py::TestEnvVarExpansionIntegration::test_env_var_expanded_in_config PASSED [ 10%] tests/test_cli_integration.py::TestEnvVarExpansionIntegration::test_multiple_env_vars_expanded PASSED [ 13%] tests/test_cli_integration.py::TestEnvVarExpansionIntegration::test_env_var_with_default_uses_default_when_unset PASSED [ 17%] tests/test_cli_integration.py::TestEnvVarExpansionIntegration::test_env_var_with_default_uses_value_when_set PASSED [ 20%] tests/test_cli_integration.py::TestEnvVarExpansionIntegration::test_missing_env_var_without_default_exits_with_error PASSED [ 24%] tests/test_cli_integration.py::TestEnvVarExpansionIntegration::test_env_var_in_machine_config_section PASSED [ 27%] tests/test_config.py::TestExpandEnvVars::test_plain_string_unchanged PASSED [ 31%] tests/test_config.py::TestExpandEnvVars::test_non_string_types_unchanged PASSED [ 34%] tests/test_config.py::TestExpandEnvVars::test_simple_variable_substitution PASSED [ 37%] tests/test_config.py::TestExpandEnvVars::test_variable_embedded_in_string PASSED [ 41%] tests/test_config.py::TestExpandEnvVars::test_multiple_variables_in_string PASSED [ 44%] tests/test_config.py::TestExpandEnvVars::test_default_value_when_var_unset PASSED [ 48%] tests/test_config.py::TestExpandEnvVars::test_default_value_ignored_when_var_set PASSED [ 51%] tests/test_config.py::TestExpandEnvVars::test_default_value_empty_string PASSED [ 55%] tests/test_config.py::TestExpandEnvVars::test_unset_variable_without_default_exits PASSED [ 58%] tests/test_config.py::TestExpandEnvVars::test_dict_values_expanded PASSED [ 62%] tests/test_config.py::TestExpandEnvVars::test_nested_dict_expanded PASSED [ 65%] tests/test_config.py::TestExpandEnvVars::test_list_values_expanded PASSED [ 68%] tests/test_config.py::TestExpandEnvVars::test_mixed_nested_structure PASSED [ 72%] tests/test_input_validation.py::TestInvalidMachineType::test_invalid_machine_type_error_message PASSED [ 75%] tests/test_input_validation.py::TestInvalidMachineType::test_valid_machine_type_accepted PASSED [ 79%] tests/test_input_validation.py::TestMissingConfigSections::test_missing_provider_section PASSED [ 82%] tests/test_input_validation.py::TestMissingConfigSections::test_missing_access_token PASSED [ 86%] tests/test_input_validation.py::TestMissingConfigSections::test_missing_ssh_key PASSED [ 89%] tests/test_input_validation.py::TestMissingConfigSections::test_missing_machines_section PASSED [ 93%] tests/test_input_validation.py::TestMissingConfigSections::test_missing_new_user_name_in_machine PASSED [ 96%] tests/test_input_validation.py::TestCreateNoInitialize::test_no_initialize_without_type_no_crash PASSED [100%] ============================== 29 passed in 2.47s ==============================
diff --git a/machine/providers/vultr.py b/machine/providers/vultr.py
index 42785d5..8f5cbe8 100644
--- a/machine/providers/vultr.py
+++ b/machine/providers/vultr.py
@@ -1,4 +1,5 @@
import base64
+import time
from vultr import Vultr, VultrException
@@ -64,11 +65,23 @@ class VultrProvider(CloudProvider):
return _instance_to_vm(result)
def destroy_vm(self, vm_id) -> bool:
- try:
- self._client.delete_instance(vm_id)
- except VultrException as e:
- fatal_error(f"Error: machine with id {vm_id} not found: {e}")
- return True
+ # Vultr returns HTTP 500 if the instance is still pending or locked
+ # (e.g. during provisioning). Retry deletion with backoff.
+ for attempt in range(24):
+ try:
+ self._client.delete_instance(vm_id)
+ return True
+ except VultrException as e:
+ error_msg = str(e)
+ if "500" in error_msg and ("not currently active" in error_msg or "currently locked" in error_msg):
+ info("Waiting for instance to become ready before destroying...")
+ time.sleep(5)
+ elif "404" in error_msg:
+ return True # already gone
+ else:
+ fatal_error(f"Error: machine with id {vm_id} not found: {e}")
+ fatal_error(f"Error: timed out waiting to destroy instance {vm_id}")
+ return False
def list_vms(self, tag=None) -> list:
try:
diff --git a/machine/subcommands/create.py b/machine/subcommands/create.py
index 4995f40..95d2ec9 100644
--- a/machine/subcommands/create.py
+++ b/machine/subcommands/create.py
@@ -83,12 +83,14 @@ def command(context, name, tag, type, region, machine_size, image, wait_for_ip,
info(f"Assigned droplet to project: {config.project}")
# If requested, or if we are going to set a DNS record get the VM's IPv4 address
- ip_address = vm.ip_address
+ # Vultr returns "0.0.0.0" as main_ip while the instance is still pending,
+ # so treat that the same as no IP assigned yet.
+ ip_address = vm.ip_address if vm.ip_address != "0.0.0.0" else None
if (wait_for_ip or update_dns) and not ip_address:
while not ip_address:
time.sleep(1)
vm = provider.get_vm(vm.id)
- ip_address = vm.ip_address
+ ip_address = vm.ip_address if vm.ip_address != "0.0.0.0" else None
if d.opt.verbose:
output("Waiting for droplet IP address")
if d.opt.quiet:
diff --git a/tests/test_e2e.py b/tests/test_e2e.py
index 29cc0fe..176559d 100644
--- a/tests/test_e2e.py
+++ b/tests/test_e2e.py
@@ -235,7 +235,7 @@ def instance(config_file, session_id):
yield info
# ---- TEARDOWN: destroy with DNS cleanup --------------------------------
- run_machine(
+ destroy_result = run_machine(
"--verbose",
"destroy",
"--no-confirm",
@@ -244,6 +244,10 @@ def instance(config_file, session_id):
config_file=config_file,
session_id=session_id,
)
+ if destroy_result.returncode != 0:
+ print(f"TEARDOWN WARNING: destroy exited {destroy_result.returncode}", flush=True)
+ print(f" stdout: {destroy_result.stdout}", flush=True)
+ print(f" stderr: {destroy_result.stderr}", flush=True)
# ---------------------------------------------------------------------------
@@ -288,7 +292,9 @@ class TestInstanceLifecycle:
instances = json.loads(result.stdout)
matched = [i for i in instances if str(i["id"]) == instance["id"]]
assert len(matched) == 1
- assert matched[0]["ip"] is not None, "Instance has no IP address"
+ ip = matched[0]["ip"]
+ assert ip is not None, "Instance has no IP address"
+ assert ip != "0.0.0.0", "Instance IP is 0.0.0.0 (not yet assigned)"
def test_dns_record_created(self, instance, config_file, session_id):
"""Verify that a DNS A record was created for the instance."""Here's a summary of the three bugs found and fixed:
"0.0.0.0" treated as valid IP (create.py) — Issues #55, #53, #61Vultr returns main_ip: "0.0.0.0" for newly-created instances while they're still pending. The code not ip_address evaluates "0.0.0.0" as truthy, so the wait-for-IP loop was skipped entirely. The DNS record was then created with 0.0.0.0, and the list-domain command couldn't match it to the VM (which eventually gets a real IP), causing the DNS test to fail.
Fix: Treat "0.0.0.0" the same as None when checking the IP address.
vultr.py) — Issue #57Vultr returns HTTP 500 with two different messages when you try to delete an instance that isn't ready: - "This subscription is not currently active" (status: pending) - "Server is currently locked" (status: active but still provisioning)
The old code called fatal_error on any exception, so the teardown subprocess would exit non-zero and the VM was never cleaned up. Before the IP fix, the create command returned in ~2 seconds (skipping the wait), so the VM was almost always still locked during teardown.
Fix: Retry deletion up to 24 times (2 min) with 5s backoff for known transient 500 errors.
test_e2e.py)test_instance_has_ip only checked ip is not None — "0.0.0.0" passed this check.
Fix: Also assert ip != "0.0.0.0". Plus added teardown diagnostics so future destroy failures are visible.
Thanks! To recap the key findings:
0.0.0.0 as the initial IP for pending instances, unlike DigitalOcean which returns empty/null — this one issue cascaded into all four Vultr issuesBoth are Vultr-specific API behaviors that the original code didn't account for. The fixes are minimal and targeted — just the IP sentinel check in create.py and retry logic in vultr.py's destroy_vm.
{"body":"","number":62,"title":"Fixes for various Vultr API issues","url":"https://github.com/stirlingbridge/machine/pull/62"}gh pr edit 62 --body "$(cat <<'EOF'
## Summary
Fixes four open Vultr issues (#53, #55, #57, #61) which all stem from two Vultr-specific API behaviors that differ from DigitalOcean:
1. **Vultr returns `0.0.0.0` as the initial IP** for pending instances (DO returns empty/null). The `create` command treated `"0.0.0.0"` as a valid IP, skipping the wait-for-IP loop entirely. This caused DNS records to be created with `0.0.0.0`.
2. **Vultr rejects deletion of pending/locked instances** with HTTP 500 (DO allows deleting in any state). Because the IP wait was skipped, the create command returned in ~2 seconds, and the test teardown tried to destroy the VM while it was still provisioning — silently failing every time.
## Changes
- **`machine/subcommands/create.py`**: Treat `"0.0.0.0"` the same as `None` when checking the IP address, so the wait-for-IP loop runs correctly on Vultr.
- **`machine/providers/vultr.py`**: Retry `destroy_vm` with 5s backoff (up to 2 min) when Vultr returns transient 500 errors ("not currently active" or "currently locked").
- **`tests/test_e2e.py`**: Strengthen `test_instance_has_ip` to reject `0.0.0.0`. Add teardown diagnostics so future destroy failures are visible in test output.
## Test plan
- [x] All 6 Vultr e2e tests pass (previously `test_dns_record_created` always failed)
- [x] VM is fully cleaned up after test run (previously accumulated on every run)
- [x] All 29 unit tests pass
- [x] `ruff check` passes
Closes #53, closes #55, closes #57, closes #61
🤖 Generated with [Claude Code](https://claude.com/claude-code)
EOF
)" 2>&1Done — PR #62 has been updated with the full diagnosis and fix details: https://github.com/stirlingbridge/machine/pull/62