#!/usr/bin/env perl # # host_manager.pl - Minimal host registry web app with no CPAN dependencies. # use strict; use warnings; use Cwd qw(abs_path); use DBI; use Digest::SHA qw(hmac_sha1 hmac_sha256_hex sha256_hex); use File::Basename qw(dirname); use File::Path qw(make_path); use IO::Socket::INET; use POSIX qw(strftime); use Time::HiRes qw(time); my $script_dir = dirname(abs_path($0)); my $project_dir = dirname($script_dir); my %opt = ( bind => $ENV{HOST_MANAGER_BIND} || '127.0.0.1', port => $ENV{HOST_MANAGER_PORT} || 8088, db => $ENV{HOST_MANAGER_DB} || "$project_dir/var/host-manager.sqlite", data => $ENV{HOST_MANAGER_DATA} || "$project_dir/config/hosts.yaml", local_hosts_tsv => $ENV{HOST_MANAGER_LOCAL_HOSTS_TSV} || "$project_dir/config/local-hosts.tsv", dns_publish_trigger => $ENV{HOST_MANAGER_DNS_PUBLISH_TRIGGER} || "$project_dir/var/dns-publish.trigger", work_orders => $ENV{HOST_MANAGER_WORK_ORDERS} || "$project_dir/config/work-orders.yaml", ); my $print_local_hosts_tsv = 0; while (@ARGV) { my $arg = shift @ARGV; if ($arg eq '--bind') { $opt{bind} = shift @ARGV; } elsif ($arg eq '--port') { $opt{port} = shift @ARGV; } elsif ($arg eq '--db') { $opt{db} = shift @ARGV; } elsif ($arg eq '--data') { $opt{data} = shift @ARGV; } elsif ($arg eq '--local-hosts-tsv') { $opt{local_hosts_tsv} = shift @ARGV; } elsif ($arg eq '--work-orders') { $opt{work_orders} = shift @ARGV; } elsif ($arg eq '--print-local-hosts-tsv') { $print_local_hosts_tsv = 1; } elsif ($arg eq '--help' || $arg eq '-h') { usage(); exit 0; } else { die "Unknown option: $arg\n"; } } if ($print_local_hosts_tsv) { print render_local_hosts_tsv(load_registry()); exit 0; } my $session_secret = $ENV{HOST_MANAGER_SESSION_SECRET} || random_hex(32); my %sessions; my $server = IO::Socket::INET->new( LocalHost => $opt{bind}, LocalPort => $opt{port}, Proto => 'tcp', Listen => 10, ReuseAddr => 1, ) or die "Cannot listen on $opt{bind}:$opt{port}: $!\n"; print "host-manager listening on http://$opt{bind}:$opt{port}\n"; print "database: $opt{db}\n"; print "seed/export hosts file: $opt{data}\n"; print "OTP login: " . ($ENV{HOST_MANAGER_TOTP_SECRET} ? "enabled\n" : "disabled; set HOST_MANAGER_TOTP_SECRET\n"); while (my $client = $server->accept) { eval { $client->autoflush(1); handle_client($client); }; if ($@) { eval { send_json($client, 500, { error => 'internal_error', detail => "$@" }); }; } close $client; } sub usage { print <<"EOF"; Usage: perl scripts/host_manager.pl [--bind 127.0.0.1] [--port 8088] Environment: HOST_MANAGER_TOTP_SECRET Base32 TOTP secret required for write access. HOST_MANAGER_SESSION_SECRET Optional session signing secret. HOST_MANAGER_DHCP_PUSH_TOKEN Token for DHCP lease push collector. HOST_MANAGER_DB Defaults to var/host-manager.sqlite. HOST_MANAGER_DATA Defaults to config/hosts.yaml. HOST_MANAGER_LOCAL_HOSTS_TSV Defaults to config/local-hosts.tsv. HOST_MANAGER_DNS_PUBLISH_TRIGGER Defaults to var/dns-publish.trigger. HOST_MANAGER_WORK_ORDERS Defaults to config/work-orders.yaml. --print-local-hosts-tsv Print the runtime DNS manifest and exit. SQLite is the runtime source of truth. YAML files seed a new database and remain download/export compatibility artifacts. The nginx vhost keeps registry, CA, work order and download endpoints behind OTP. EOF } sub handle_client { my ($client) = @_; my $request_line = <$client>; return unless defined $request_line; $request_line =~ s/\r?\n$//; my ($method, $target) = $request_line =~ m{^([A-Z]+)\s+(\S+)\s+HTTP/}; return send_text($client, 400, 'bad request') unless $method && $target; my %headers; while (my $line = <$client>) { $line =~ s/\r?\n$//; last if $line eq ''; my ($k, $v) = split /:\s*/, $line, 2; $headers{lc $k} = $v if defined $k && defined $v; } my $body = ''; if (($headers{'content-length'} || 0) > 0) { read($client, $body, int($headers{'content-length'})); } my ($path, $query) = split /\?/, $target, 2; my %query = parse_params($query || ''); if ($method eq 'GET' && app_page_path($path)) { return send_html($client, 200, app_html()); } if ($method eq 'GET' && $path eq '/healthz') { return send_json($client, 200, { ok => json_bool(1) }); } if ($method eq 'GET' && $path eq '/api/session') { return send_json($client, 200, { authenticated => is_authenticated(\%headers) ? json_bool(1) : json_bool(0) }); } if ($method eq 'POST' && $path eq '/api/login') { return send_json($client, 503, { error => 'otp_not_configured' }) unless $ENV{HOST_MANAGER_TOTP_SECRET}; my $payload = request_payload(\%headers, $body); my $otp = $payload->{otp} || ''; if (!verify_totp($ENV{HOST_MANAGER_TOTP_SECRET} || '', $otp)) { return send_json($client, 401, { error => 'invalid_otp' }); } my $token = create_session(); return send_json($client, 200, { ok => json_bool(1) }, [ "Set-Cookie: hm_session=$token; HttpOnly; SameSite=Strict; Path=/" ]); } if ($method eq 'POST' && $path eq '/api/logout') { expire_session(\%headers); return send_json($client, 200, { ok => json_bool(1) }, [ "Set-Cookie: hm_session=deleted; Max-Age=0; Path=/" ]); } if ($method eq 'POST' && $path eq '/api/collect/dhcp-leases') { return collect_dhcp_leases($client, \%headers, $body); } return send_json($client, 401, { error => 'authentication_required' }) unless is_authenticated(\%headers); if ($method eq 'GET' && $path eq '/api/hosts') { my $registry = load_registry(); return send_json($client, 200, registry_payload($registry)); } if ($method eq 'GET' && $path eq '/api/work-orders') { return send_json($client, 200, work_orders_payload(load_work_orders())); } if ($method eq 'GET' && $path eq '/api/debug/database/tables') { return send_json($client, 200, debug_database_tables_payload()); } if ($method eq 'GET' && $path eq '/api/debug/database/table') { return send_json($client, 200, debug_database_table_payload($query{name} || $query{table} || '', $query{limit} || 100)); } if ($method eq 'GET' && $path eq '/download/debug/database/table.json') { my $export = debug_database_table_export_payload($query{name} || $query{table} || ''); return send_json($client, 400, { error => $export->{error} }) if $export->{error}; return send_download($client, 200, json_encode($export), 'application/json; charset=utf-8', debug_table_export_filename($export->{table}, 'json')); } if ($method eq 'GET' && $path eq '/download/debug/database/table.csv') { my $export = debug_database_table_export_payload($query{name} || $query{table} || ''); return send_json($client, 400, { error => $export->{error} }) if $export->{error}; return send_download($client, 200, render_debug_table_csv($export), 'text/csv; charset=utf-8', debug_table_export_filename($export->{table}, 'csv')); } if ($method eq 'GET' && $path eq '/download/hosts.yaml') { my $registry = load_registry(); return send_download($client, 200, render_hosts_yaml($registry), 'application/x-yaml; charset=utf-8', 'hosts.yaml'); } if ($method eq 'GET' && $path eq '/download/local-hosts.tsv') { my $registry = load_registry(); return send_download($client, 200, render_local_hosts_tsv($registry), 'text/tab-separated-values; charset=utf-8', 'local-hosts.tsv'); } if ($method eq 'GET' && $path eq '/download/monitoring.json') { my $registry = load_registry(); return send_download($client, 200, json_encode(render_monitoring($registry)), 'application/json; charset=utf-8', 'monitoring-hosts.json'); } if ($method eq 'GET' && $path eq '/api/ca/status') { return send_json_raw($client, 200, ca_manager_json('status-json')); } if ($method eq 'GET' && $path eq '/api/ca/certificates') { return send_json_raw($client, 200, ca_manager_json('list-json')); } if ($method eq 'GET' && $path eq '/download/ca.crt') { return send_file($client, ca_cert_path(), 'application/x-pem-file; charset=utf-8', 'xdev-madagascar-host-ca.crt'); } if ($method eq 'GET' && $path =~ m{\A/download/ca/cert/([A-Za-z0-9_.-]+)\.crt\z}) { my $name = $1; return send_file($client, ca_issued_cert_path($name), 'application/x-pem-file; charset=utf-8', "$name.crt"); } if ($method eq 'GET' && $path =~ m{\A/download/ca/key/([A-Za-z0-9_.-]+)\.key\z}) { my $name = $1; return send_file($client, ca_issued_key_path($name), 'application/x-pem-file; charset=utf-8', "$name.key"); } if ($method eq 'POST' && $path =~ m{^/api/}) { if ($path eq '/api/hosts/upsert') { my $payload = request_payload(\%headers, $body); return upsert_host($client, $payload); } if ($path eq '/api/hosts/delete') { my $payload = request_payload(\%headers, $body); return delete_host($client, $payload->{id} || ''); } if ($path eq '/api/hosts/certificate') { my $payload = request_payload(\%headers, $body); return set_host_certificate($client, $payload); } if ($path eq '/api/hosts/issue-certificate') { my $payload = request_payload(\%headers, $body); return issue_host_certificate($client, $payload); } if ($path eq '/api/vhosts/reassign') { my $payload = request_payload(\%headers, $body); return reassign_vhost($client, $payload); } if ($path eq '/api/vhosts/upsert') { my $payload = request_payload(\%headers, $body); return upsert_vhost($client, $payload); } if ($path eq '/api/vhosts/delete') { my $payload = request_payload(\%headers, $body); return delete_vhost($client, $payload); } if ($path eq '/api/vhosts/certificate') { my $payload = request_payload(\%headers, $body); return set_vhost_certificate($client, $payload); } if ($path eq '/api/vhosts/issue-certificate') { my $payload = request_payload(\%headers, $body); return issue_vhost_certificate($client, $payload); } if ($path eq '/api/work-orders/confirm') { my $payload = request_payload(\%headers, $body); return confirm_work_order($client, $payload); } if ($path eq '/api/work-orders/checklist') { my $payload = request_payload(\%headers, $body); return update_work_order_checklist($client, $payload); } if ($path eq '/api/render/local-hosts-tsv') { my $registry = load_registry(); my $publish = publish_dns_change($registry, 'manual-render'); return send_json($client, 200, { ok => json_bool(1), file => $opt{local_hosts_tsv}, dns_publish => $publish }); } } return send_json($client, 404, { error => 'not_found' }); } sub app_page_path { my ($path) = @_; return $path =~ m{\A/(?:|overview|hosts|vhosts|dns|work-orders|ca|debug)\z}; } sub load_registry { my $registry = load_registry_from_db(); normalize_registry_policy($registry); return $registry; } sub save_registry { my ($registry) = @_; $registry->{updated_at} = iso_now(); normalize_registry_policy($registry); save_registry_to_db($registry); return publish_dns_change($registry, 'registry-save'); } sub load_work_orders { return load_work_orders_from_db(); } sub save_work_orders { my ($orders) = @_; save_work_orders_to_db($orders); } sub work_orders_payload { my ($orders) = @_; my $pending = 0; for my $wo (@{ $orders->{work_orders} || [] }) { $pending++ if ($wo->{status} || 'pending') eq 'pending'; } return { version => $orders->{version}, work_orders => $orders->{work_orders} || [], counts => { work_orders => scalar @{ $orders->{work_orders} || [] }, pending => $pending, }, }; } sub collect_dhcp_leases { my ($client, $headers, $body) = @_; my $expected = $ENV{HOST_MANAGER_DHCP_PUSH_TOKEN} || ''; return send_json($client, 503, { error => 'dhcp_push_not_configured' }) unless length $expected; my $provided = dhcp_push_token_from_headers($headers); return send_json($client, 401, { error => 'invalid_dhcp_push_token' }) unless token_matches($expected, $provided); my $payload = request_payload($headers, $body); my @leases = dhcp_payload_leases($payload); return send_json($client, 400, { error => 'missing_dhcp_leases' }) unless @leases; my $dbh = dbh(); my $now = iso_now(); my $worker_id = clean_id($payload->{worker_id} || $payload->{source_id} || 'dhcp-router'); $worker_id ||= 'dhcp-router'; my @stored; with_transaction($dbh, sub { upsert_dhcp_worker($dbh, $worker_id, $now); for my $lease (@leases) { my $stored = upsert_dhcp_lease($dbh, $worker_id, $lease, $now); push @stored, $stored if $stored; } $dbh->do( 'UPDATE data_workers SET last_run_at = ?, updated_at = ? WHERE worker_id = ?', undef, $now, $now, $worker_id, ); }); return send_json($client, 200, { ok => json_bool(1), worker_id => $worker_id, stored => scalar(@stored), leases => \@stored, }); } sub dhcp_push_token_from_headers { my ($headers) = @_; my $token = clean_scalar($headers->{'x-dhcp-push-token'} || ''); return $token if length $token; my $authorization = clean_scalar($headers->{authorization} || ''); return $1 if $authorization =~ /\ABearer\s+(.+)\z/i; return ''; } sub token_matches { my ($expected, $provided) = @_; return 0 unless length($expected || '') && length($provided || ''); return 0 unless length($expected) == length($provided); my $diff = 0; for my $i (0 .. length($expected) - 1) { $diff |= ord(substr($expected, $i, 1)) ^ ord(substr($provided, $i, 1)); } return $diff == 0 ? 1 : 0; } sub dhcp_payload_leases { my ($payload) = @_; return () unless ref($payload) eq 'HASH'; if (ref($payload->{leases}) eq 'ARRAY') { return grep { ref($_) eq 'HASH' } @{ $payload->{leases} }; } return ($payload); } sub upsert_dhcp_worker { my ($dbh, $worker_id, $now) = @_; $dbh->do( 'INSERT INTO data_workers (worker_id, worker_type, name, status, source, last_run_at, notes, created_at, updated_at) ' . "VALUES (?, 'dhcp', 'Router DHCP leases', 'active', 'push:192.168.2.1', ?, 'DHCP lease push collector source.', ?, ?) " . 'ON CONFLICT(worker_id) DO UPDATE SET worker_type = excluded.worker_type, name = excluded.name, status = excluded.status, ' . 'source = excluded.source, last_run_at = excluded.last_run_at, notes = excluded.notes, updated_at = excluded.updated_at', undef, $worker_id, $now, $now, $now, ); } sub upsert_dhcp_lease { my ($dbh, $worker_id, $lease, $now) = @_; my $ip = clean_ip($lease->{ip_address} || $lease->{ip} || $lease->{address} || ''); my $mac = clean_mac($lease->{mac_address} || $lease->{mac} || $lease->{active_mac} || ''); return unless length $ip || length $mac; my $name = normalize_dhcp_name($lease->{observed_name} || $lease->{host_name} || $lease->{hostname} || $lease->{name} || ''); my $state = clean_scalar($lease->{lease_state} || $lease->{state} || $lease->{status} || ''); if (!length $state && exists $lease->{bound}) { $state = ($lease->{bound} || '') eq '1' ? 'bound' : 'unbound'; } $state ||= 'observed'; my $lease_key = length $mac ? "$worker_id|mac|$mac" : "$worker_id|ip|$ip"; my $host_fqdn = match_dhcp_host_fqdn($dbh, $name, $ip); my $raw = json_encode($lease); $dbh->do( 'INSERT INTO dhcp_leases (lease_key, worker_id, host_fqdn, observed_name, ip_address, mac_address, lease_state, first_seen, last_seen, raw) ' . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ' . 'ON CONFLICT(lease_key) DO UPDATE SET host_fqdn = excluded.host_fqdn, observed_name = excluded.observed_name, ' . 'ip_address = excluded.ip_address, mac_address = excluded.mac_address, lease_state = excluded.lease_state, ' . 'last_seen = excluded.last_seen, raw = excluded.raw', undef, $lease_key, $worker_id, length($host_fqdn) ? $host_fqdn : undef, $name, $ip, $mac, $state, $now, $now, $raw, ); return { lease_key => $lease_key, host_fqdn => $host_fqdn, observed_name => $name, ip_address => $ip, mac_address => $mac, lease_state => $state, }; } sub match_dhcp_host_fqdn { my ($dbh, $name, $ip) = @_; my @names; $name = normalize_dns_name($name || ''); if (length $name) { push @names, $name; push @names, "$name.madagascar.xdev.ro" unless $name =~ /\./; } for my $candidate (unique_preserve(@names)) { my ($fqdn) = $dbh->selectrow_array('SELECT fqdn FROM hosts WHERE fqdn = ? AND status <> ?', undef, $candidate, 'retired'); return $fqdn if $fqdn; ($fqdn) = $dbh->selectrow_array('SELECT host_fqdn FROM host_aliases WHERE alias_name = ? AND status = ?', undef, $candidate, 'active'); return $fqdn if $fqdn; } if (length($ip || '')) { my ($fqdn) = $dbh->selectrow_array('SELECT fqdn FROM hosts WHERE (dns_ip = ? OR hosts_ip = ?) AND status <> ? ORDER BY fqdn LIMIT 1', undef, $ip, $ip, 'retired'); return $fqdn if $fqdn; } return ''; } sub confirm_work_order { my ($client, $payload) = @_; my $id = clean_scalar($payload->{id} || ''); return send_json($client, 400, { error => 'invalid_work_order_id' }) unless $id =~ /\AWO-[A-Za-z0-9_.-]+\z/; return send_json($client, 400, { error => 'confirmation_required' }) unless clean_scalar($payload->{confirm} || '') eq $id; my $orders = load_work_orders(); my $work_order; for my $wo (@{ $orders->{work_orders} || [] }) { if (($wo->{id} || '') eq $id) { $work_order = $wo; last; } } return send_json($client, 404, { error => 'work_order_not_found' }) unless $work_order; return send_json($client, 409, { error => 'work_order_not_pending' }) unless ($work_order->{status} || 'pending') eq 'pending'; my $incomplete = incomplete_work_order_items($work_order); return send_json($client, 409, { error => 'work_order_incomplete', incomplete => $incomplete, }) if @$incomplete; my $registry = load_registry(); my $results = apply_work_order($registry, $work_order); $work_order->{status} = 'confirmed'; $work_order->{confirmed_at} = iso_now(); $work_order->{result} = scalar(@$results) . ' action(s) applied'; my $publish = save_registry($registry); save_work_orders($orders); return send_json($client, 200, { ok => json_bool(1), work_order => $work_order, results => $results, local_hosts_tsv => $opt{local_hosts_tsv}, dns_publish => $publish, }); } sub update_work_order_checklist { my ($client, $payload) = @_; my $id = clean_scalar($payload->{id} || ''); my $item_id = clean_scalar($payload->{item_id} || ''); my $status = clean_scalar($payload->{status} || ''); my $notes = clean_scalar($payload->{notes} || ''); return send_json($client, 400, { error => 'invalid_work_order_id' }) unless $id =~ /\AWO-[A-Za-z0-9_.-]+\z/; return send_json($client, 400, { error => 'invalid_checklist_item' }) unless $item_id =~ /\A[A-Za-z0-9_.-]+\z/; return send_json($client, 400, { error => 'invalid_checklist_status' }) unless $status =~ /\A(?:pending|done|blocked)\z/; my $orders = load_work_orders(); my $work_order; for my $wo (@{ $orders->{work_orders} || [] }) { if (($wo->{id} || '') eq $id) { $work_order = $wo; last; } } return send_json($client, 404, { error => 'work_order_not_found' }) unless $work_order; return send_json($client, 409, { error => 'work_order_not_pending' }) unless ($work_order->{status} || 'pending') eq 'pending'; my $item; for my $candidate (@{ $work_order->{checklist} || [] }) { if (($candidate->{id} || '') eq $item_id) { $item = $candidate; last; } } return send_json($client, 404, { error => 'checklist_item_not_found' }) unless $item; $item->{status} = $status; $item->{updated_at} = iso_now(); $item->{notes} = $notes if length $notes; save_work_orders($orders); return send_json($client, 200, { ok => json_bool(1), work_order => $work_order }); } sub incomplete_work_order_items { my ($work_order) = @_; my @incomplete; for my $item (@{ $work_order->{checklist} || [] }) { push @incomplete, $item unless ($item->{status} || 'pending') eq 'done'; } return \@incomplete; } sub apply_work_order { my ($registry, $work_order) = @_; my @results; for my $action (@{ $work_order->{actions} || [] }) { my $type = $action->{type} || ''; if ($type eq 'remove_name') { my $host_id = $action->{host_id} || ''; my $name = $action->{name} || ''; my $removed = 0; for my $host (@{ $registry->{hosts} || [] }) { next unless ($host->{id} || '') eq $host_id; my @kept_aliases = grep { $_ ne $name } declared_alias_names($host); my @kept_vhosts = grep { $_ ne $name } declared_vhost_names($host); $removed = (@kept_aliases != @{ $host->{aliases} || [] }) || (@kept_vhosts != @{ $host->{vhosts} || [] }); $host->{aliases} = \@kept_aliases; $host->{vhosts} = \@kept_vhosts; last; } push @results, { type => $type, host_id => $host_id, name => $name, removed => json_bool($removed), }; } else { die "Unsupported work order action: $type\n"; } } return \@results; } sub registry_payload { my ($registry) = @_; my $problems = analyze_hosts($registry->{hosts}); my $dbh = dbh(); my %host_tls = host_tls_payloads($dbh); my @hosts = map { host_payload($_, $host_tls{ canonical_host_fqdn($_) }) } @{ $registry->{hosts} }; my @vhosts = vhost_payloads($dbh); my @certificates = certificate_payloads($dbh); my $vhost_count = sum(map { scalar declared_vhost_names($_) } @{ $registry->{hosts} }); return { version => $registry->{version}, updated_at => $registry->{updated_at}, policy => $registry->{policy}, hosts => \@hosts, vhosts => \@vhosts, certificates => \@certificates, problems => $problems, counts => { hosts => scalar @{ $registry->{hosts} }, vhosts => scalar(@vhosts) || $vhost_count, problems => scalar @$problems, }, }; } sub host_tls_payloads { my ($dbh) = @_; my %rows; my $sth = $dbh->prepare(<<'SQL'); SELECT ht.host_fqdn, ht.certificate_id, c.common_name, c.not_after, c.fingerprint_sha256, c.status AS certificate_status FROM host_tls ht LEFT JOIN certificates c ON c.certificate_id = ht.certificate_id ORDER BY ht.host_fqdn SQL $sth->execute; while (my $row = $sth->fetchrow_hashref) { my $host_fqdn = clean_scalar($row->{host_fqdn} || ''); next unless length $host_fqdn; my $cert_id = clean_scalar($row->{certificate_id} || ''); my %payload = ( certificate_id => $cert_id, ); if (length $cert_id) { $payload{certificate} = { id => $cert_id, name => $cert_id, common_name => clean_scalar($row->{common_name} || ''), status => clean_scalar($row->{certificate_status} || ''), not_after => clean_scalar($row->{not_after} || ''), fingerprint_sha256 => clean_scalar($row->{fingerprint_sha256} || ''), has_private_key => json_bool(ca_private_key_exists($cert_id)), }; } $rows{$host_fqdn} = \%payload; } return %rows; } sub vhost_payloads { my ($dbh) = @_; my @rows; my $sth = $dbh->prepare(<<'SQL'); SELECT v.vhost_fqdn, v.host_fqdn, v.status AS vhost_status, v.certificate_id, h.legacy_id, h.monitoring, h.status AS host_status, c.common_name, c.not_after, c.fingerprint_sha256, c.status AS certificate_status FROM vhosts v JOIN hosts h ON h.fqdn = v.host_fqdn LEFT JOIN certificates c ON c.certificate_id = v.certificate_id WHERE v.status = 'active' ORDER BY v.vhost_fqdn SQL $sth->execute; while (my $row = $sth->fetchrow_hashref) { my $cert_id = clean_scalar($row->{certificate_id} || ''); my %certificate = $cert_id ? ( id => $cert_id, name => $cert_id, common_name => clean_scalar($row->{common_name} || ''), status => clean_scalar($row->{certificate_status} || ''), not_after => clean_scalar($row->{not_after} || ''), fingerprint_sha256 => clean_scalar($row->{fingerprint_sha256} || ''), has_private_key => json_bool(ca_private_key_exists($cert_id)), ) : (); push @rows, { vhost => $row->{vhost_fqdn}, vhost_fqdn => $row->{vhost_fqdn}, host_id => $row->{legacy_id} || '', host_fqdn => $row->{host_fqdn}, derived_aliases => short_alias_for_fqdn($row->{vhost_fqdn}) ? [ short_alias_for_fqdn($row->{vhost_fqdn}) ] : [], monitoring => $row->{monitoring} || '', status => $row->{host_status} || $row->{vhost_status} || '', vhost_status => $row->{vhost_status} || '', certificate_id => $cert_id, certificate => $cert_id ? \%certificate : undef, }; } return @rows; } sub certificate_payloads { my ($dbh) = @_; my @certificates; my $sth = $dbh->prepare('SELECT * FROM certificates WHERE status <> ? ORDER BY certificate_id'); $sth->execute('retired'); while (my $row = $sth->fetchrow_hashref) { my $id = clean_scalar($row->{certificate_id} || ''); next unless $id; push @certificates, { id => $id, name => $id, host_fqdn => $row->{host_fqdn} || '', common_name => $row->{common_name} || '', subject => $row->{subject} || '', issuer => $row->{issuer} || '', serial => $row->{serial} || '', status => $row->{status} || '', not_before => $row->{not_before} || '', not_after => $row->{not_after} || '', fingerprint_sha256 => $row->{fingerprint_sha256} || '', dns_names => [ certificate_dns_names($dbh, $id) ], has_private_key => json_bool(ca_private_key_exists($id)), }; } return @certificates; } sub certificate_dns_names { my ($dbh, $certificate_id) = @_; my @names; my $sth = $dbh->prepare('SELECT dns_name FROM certificate_dns_names WHERE certificate_id = ? ORDER BY dns_name'); $sth->execute($certificate_id); while (my ($name) = $sth->fetchrow_array) { push @names, $name; } return @names; } sub upsert_host { my ($client, $payload) = @_; my $id = clean_id($payload->{id} || ''); return send_json($client, 400, { error => 'invalid_id' }) unless $id; my $ip = canonical_ip($payload); return send_json($client, 400, { error => 'missing_ip' }) unless $ip; my $fqdn = canonical_host_fqdn($payload); return send_json($client, 400, { error => 'missing_fqdn' }) unless $fqdn; my @aliases = clean_alias_names($payload); my $registry = load_registry(); my ($existing_host) = grep { ($_->{id} || '') eq $id } @{ $registry->{hosts} || [] }; my @vhosts = defined $payload->{vhosts} ? clean_vhost_names($payload) : ($existing_host ? declared_vhost_names($existing_host) : ()); my %host = ( id => $id, fqdn => $fqdn, status => clean_scalar($payload->{status} || 'active'), ip => $ip, aliases => \@aliases, vhosts => \@vhosts, roles => [ clean_list($payload->{roles}) ], sources => [ clean_list($payload->{sources}) ], monitoring => clean_scalar($payload->{monitoring} || 'pending'), notes => clean_scalar($payload->{notes} || ''), ); my $response = eval { my $replaced = 0; for my $i (0 .. $#{ $registry->{hosts} }) { if ($registry->{hosts}->[$i]{id} eq $id) { $registry->{hosts}->[$i] = \%host; $replaced = 1; last; } } push @{ $registry->{hosts} }, \%host unless $replaced; save_registry($registry); 1; }; if (!$response) { my $err = $@ || 'upsert_failed'; return send_json($client, 409, { error => 'alias_conflict', detail => clean_scalar($err) }) if $err =~ /alias_conflict:/; die $err; } return send_json($client, 200, { ok => json_bool(1), host => \%host }); } sub delete_host { my ($client, $id) = @_; $id = clean_id($id); return send_json($client, 400, { error => 'invalid_id' }) unless $id; my $registry = load_registry(); my @kept = grep { $_->{id} ne $id } @{ $registry->{hosts} }; return send_json($client, 404, { error => 'not_found' }) if @kept == @{ $registry->{hosts} }; $registry->{hosts} = \@kept; save_registry($registry); return send_json($client, 200, { ok => json_bool(1) }); } sub reassign_vhost { my ($client, $payload) = @_; my $vhost = normalize_dns_name($payload->{vhost_fqdn} || ''); my $target_fqdn = normalize_dns_name($payload->{host_fqdn} || $payload->{fqdn} || ''); return send_json($client, 400, { error => 'invalid_vhost' }) unless vhost_name_is_valid($vhost); return send_json($client, 400, { error => 'missing_target_host' }) unless $target_fqdn; my $dbh = dbh(); my ($current_fqdn) = $dbh->selectrow_array( "SELECT host_fqdn FROM vhosts WHERE vhost_fqdn = ? AND status = 'active'", undef, $vhost, ); return send_json($client, 404, { error => 'vhost_not_found' }) unless $current_fqdn; return send_json($client, 400, { error => 'invalid_target_host' }) unless db_scalar($dbh, 'SELECT COUNT(*) FROM hosts WHERE fqdn = ? AND status <> ?', $target_fqdn, 'retired'); return send_json($client, 200, { ok => json_bool(1), vhost_fqdn => $vhost, host_fqdn => $current_fqdn }) if $current_fqdn eq $target_fqdn; my $result = eval { with_transaction($dbh, sub { my $now = iso_now(); $dbh->do( "UPDATE vhosts SET host_fqdn = ?, updated_at = ?, status = 'active' WHERE vhost_fqdn = ?", undef, $target_fqdn, $now, $vhost, ); my $registry = load_registry_from_db(); my ($target_host) = grep { ($_->{fqdn} || '') eq $target_fqdn } @{ $registry->{hosts} || [] }; my ($current_host) = grep { ($_->{fqdn} || '') eq $current_fqdn } @{ $registry->{hosts} || [] }; upsert_host_to_db($dbh, $target_host) if $target_host; upsert_host_to_db($dbh, $current_host) if $current_host; set_schema_meta($dbh, 'registry_updated_at', iso_now()); }); 1; }; if (!$result) { my $err = $@ || 'vhost_reassign_failed'; return send_json($client, 409, { error => 'vhost_reassign_failed', detail => clean_scalar($err) }); } my $publish = publish_dns_change(load_registry(), 'vhost-reassign'); return send_json($client, 200, { ok => json_bool(1), vhost_fqdn => $vhost, host_fqdn => $target_fqdn, previous_host_fqdn => $current_fqdn, dns_publish => $publish }); } sub upsert_vhost { my ($client, $payload) = @_; my $vhost = normalize_dns_name($payload->{vhost_fqdn} || ''); my $target_fqdn = normalize_dns_name($payload->{host_fqdn} || $payload->{fqdn} || ''); return send_json($client, 400, { error => 'invalid_vhost' }) unless vhost_name_is_valid($vhost); return send_json($client, 400, { error => 'missing_target_host' }) unless $target_fqdn; my $dbh = dbh(); return send_json($client, 400, { error => 'invalid_target_host' }) unless db_scalar($dbh, 'SELECT COUNT(*) FROM hosts WHERE fqdn = ? AND status <> ?', $target_fqdn, 'retired'); return send_json($client, 400, { error => 'vhost_matches_host' }) if db_scalar($dbh, 'SELECT COUNT(*) FROM hosts WHERE fqdn = ? AND status <> ?', $vhost, 'retired'); my ($current_fqdn) = $dbh->selectrow_array( "SELECT host_fqdn FROM vhosts WHERE vhost_fqdn = ? AND status = 'active'", undef, $vhost, ); my $result = eval { with_transaction($dbh, sub { my $now = iso_now(); upsert_vhost_to_db($dbh, $target_fqdn, $vhost, $now); my $registry = load_registry_from_db(); my ($target_host) = grep { ($_->{fqdn} || '') eq $target_fqdn } @{ $registry->{hosts} || [] }; my ($current_host) = grep { ($_->{fqdn} || '') eq ($current_fqdn || '') } @{ $registry->{hosts} || [] }; upsert_host_to_db($dbh, $target_host) if $target_host; upsert_host_to_db($dbh, $current_host) if $current_host && ($current_fqdn || '') ne $target_fqdn; set_schema_meta($dbh, 'registry_updated_at', iso_now()); }); 1; }; if (!$result) { my $err = $@ || 'vhost_upsert_failed'; return send_json($client, 409, { error => 'vhost_upsert_failed', detail => clean_scalar($err) }); } my $publish = publish_dns_change(load_registry(), 'vhost-upsert'); return send_json($client, 200, { ok => json_bool(1), vhost_fqdn => $vhost, host_fqdn => $target_fqdn, previous_host_fqdn => $current_fqdn || '', dns_publish => $publish }); } sub delete_vhost { my ($client, $payload) = @_; my $vhost = normalize_dns_name($payload->{vhost_fqdn} || ''); my $confirm = normalize_dns_name($payload->{confirm} || ''); return send_json($client, 400, { error => 'invalid_vhost' }) unless vhost_name_is_valid($vhost); return send_json($client, 400, { error => 'confirmation_required' }) unless $confirm eq $vhost; my $dbh = dbh(); my ($current_fqdn) = $dbh->selectrow_array( "SELECT host_fqdn FROM vhosts WHERE vhost_fqdn = ? AND status = 'active'", undef, $vhost, ); return send_json($client, 404, { error => 'vhost_not_found' }) unless $current_fqdn; my $result = eval { with_transaction($dbh, sub { my $now = iso_now(); $dbh->do( "UPDATE vhosts SET status = 'retired', updated_at = ? WHERE vhost_fqdn = ? AND status = 'active'", undef, $now, $vhost, ); my $registry = load_registry_from_db(); my ($current_host) = grep { ($_->{fqdn} || '') eq $current_fqdn } @{ $registry->{hosts} || [] }; upsert_host_to_db($dbh, $current_host) if $current_host; set_schema_meta($dbh, 'registry_updated_at', iso_now()); }); 1; }; if (!$result) { my $err = $@ || 'vhost_delete_failed'; return send_json($client, 409, { error => 'vhost_delete_failed', detail => clean_scalar($err) }); } my $publish = publish_dns_change(load_registry(), 'vhost-delete'); return send_json($client, 200, { ok => json_bool(1), vhost_fqdn => $vhost, previous_host_fqdn => $current_fqdn, dns_publish => $publish }); } sub set_host_certificate { my ($client, $payload) = @_; my $host_fqdn = normalize_dns_name($payload->{host_fqdn} || $payload->{fqdn} || ''); my $raw_certificate_id = clean_scalar($payload->{certificate_id} || $payload->{cert_id} || ''); my $certificate_id = clean_certificate_id($raw_certificate_id); return send_json($client, 400, { error => 'invalid_host' }) unless $host_fqdn; return send_json($client, 400, { error => 'invalid_certificate' }) if length($raw_certificate_id) && !length($certificate_id); my $dbh = dbh(); return send_json($client, 404, { error => 'host_not_found' }) unless db_scalar($dbh, "SELECT COUNT(*) FROM hosts WHERE fqdn = ? AND status = 'active'", $host_fqdn); if (length $certificate_id) { return send_json($client, 400, { error => 'invalid_certificate' }) unless db_scalar($dbh, "SELECT COUNT(*) FROM certificates WHERE certificate_id = ? AND status <> 'retired'", $certificate_id); } my $now = iso_now(); with_transaction($dbh, sub { upsert_host_tls_row($dbh, $host_fqdn, $certificate_id, $now); set_schema_meta($dbh, 'registry_updated_at', $now); }); return send_json($client, 200, { ok => json_bool(1), host_fqdn => $host_fqdn, certificate_id => $certificate_id }); } sub issue_host_certificate { my ($client, $payload) = @_; my $host_fqdn = normalize_dns_name($payload->{host_fqdn} || $payload->{fqdn} || ''); return send_json($client, 400, { error => 'invalid_host' }) unless $host_fqdn; my $registry = load_registry(); my ($host) = grep { canonical_host_fqdn($_) eq $host_fqdn } @{ $registry->{hosts} || [] }; return send_json($client, 404, { error => 'host_not_found' }) unless $host; my @dns_names = unique_preserve(grep { length $_ } ( $host_fqdn, declared_alias_names($host), derived_alias_names($host), )); my $certificate_id = clean_certificate_id($host_fqdn . '-' . strftime('%Y%m%d%H%M%S', localtime)); my $dbh = dbh(); my $issued = eval { ca_manager_output('issue', $certificate_id, @dns_names); ca_manager_json('list-json'); with_transaction($dbh, sub { my $now = iso_now(); upsert_host_tls_row($dbh, $host_fqdn, $certificate_id, $now); set_schema_meta($dbh, 'registry_updated_at', $now); }); 1; }; if (!$issued) { return send_json($client, 409, { error => 'certificate_issue_failed', detail => clean_scalar($@ || '') }); } my ($cert) = grep { ($_->{id} || '') eq $certificate_id } certificate_payloads($dbh); return send_json($client, 200, { ok => json_bool(1), host_fqdn => $host_fqdn, certificate_id => $certificate_id, certificate => $cert || { id => $certificate_id, name => $certificate_id, dns_names => \@dns_names }, }); } sub set_vhost_certificate { my ($client, $payload) = @_; my $vhost = normalize_dns_name($payload->{vhost_fqdn} || ''); my $raw_certificate_id = clean_scalar($payload->{certificate_id} || $payload->{cert_id} || ''); my $certificate_id = clean_certificate_id($raw_certificate_id); return send_json($client, 400, { error => 'invalid_vhost' }) unless vhost_name_is_valid($vhost); return send_json($client, 400, { error => 'invalid_certificate' }) if length($raw_certificate_id) && !length($certificate_id); my $dbh = dbh(); return send_json($client, 404, { error => 'vhost_not_found' }) unless db_scalar($dbh, "SELECT COUNT(*) FROM vhosts WHERE vhost_fqdn = ? AND status = 'active'", $vhost); if (length $certificate_id) { return send_json($client, 400, { error => 'invalid_certificate' }) unless db_scalar($dbh, "SELECT COUNT(*) FROM certificates WHERE certificate_id = ? AND status <> 'retired'", $certificate_id); } my $now = iso_now(); $dbh->do( 'UPDATE vhosts SET certificate_id = ?, tls_mode = ?, updated_at = ? WHERE vhost_fqdn = ? AND status = ?', undef, length($certificate_id) ? $certificate_id : undef, length($certificate_id) ? 'local-ca' : 'none', $now, $vhost, 'active', ); set_schema_meta($dbh, 'registry_updated_at', $now); return send_json($client, 200, { ok => json_bool(1), vhost_fqdn => $vhost, certificate_id => $certificate_id }); } sub issue_vhost_certificate { my ($client, $payload) = @_; my $vhost = normalize_dns_name($payload->{vhost_fqdn} || ''); return send_json($client, 400, { error => 'invalid_vhost' }) unless vhost_name_is_valid($vhost); my $dbh = dbh(); my ($host_fqdn) = $dbh->selectrow_array( "SELECT host_fqdn FROM vhosts WHERE vhost_fqdn = ? AND status = 'active'", undef, $vhost, ); return send_json($client, 404, { error => 'vhost_not_found' }) unless $host_fqdn; my @dns_names = unique_preserve(grep { length $_ } ($vhost, short_alias_for_fqdn($vhost))); my $certificate_id = clean_certificate_id($vhost . '-' . strftime('%Y%m%d%H%M%S', localtime)); my $issued = eval { ca_manager_output('issue', $certificate_id, @dns_names); ca_manager_json('list-json'); with_transaction($dbh, sub { my $now = iso_now(); $dbh->do( "UPDATE vhosts SET certificate_id = ?, tls_mode = 'local-ca', updated_at = ? WHERE vhost_fqdn = ? AND status = 'active'", undef, $certificate_id, $now, $vhost, ); set_schema_meta($dbh, 'registry_updated_at', $now); }); 1; }; if (!$issued) { return send_json($client, 409, { error => 'certificate_issue_failed', detail => clean_scalar($@ || '') }); } my ($cert) = grep { ($_->{id} || '') eq $certificate_id } certificate_payloads($dbh); return send_json($client, 200, { ok => json_bool(1), vhost_fqdn => $vhost, host_fqdn => $host_fqdn, certificate_id => $certificate_id, certificate => $cert || { id => $certificate_id, name => $certificate_id, dns_names => \@dns_names }, }); } sub analyze_hosts { my ($hosts) = @_; my @problems; my (%names, %ids); for my $host (@$hosts) { push @problems, problem($host, 'duplicate-id', "Duplicate id $host->{id}") if $ids{ $host->{id} }++; my $fqdn = canonical_host_fqdn($host); push @problems, problem($host, 'missing-fqdn', 'No madagascar.xdev.ro FQDN') unless ($fqdn =~ /\.madagascar\.xdev\.ro$/) || ($host->{status} || '') ne 'active'; my @declared = declared_dns_names($host); push @problems, problem($host, 'deprecated-vad-is', 'Deprecated vad.is.xdev.ro name present') if grep { /\.vad\.is\.xdev\.ro$/ } @declared; push @problems, problem($host, 'legacy-prefix', 'Legacy prefix should be normalized out') if grep { /^(is|vad|b)-/ } @declared; for my $name (@declared) { push @problems, problem($host, 'duplicate-name', "Duplicate name $name") if $names{$name}++; } my %declared = map { $_ => 1 } @declared; for my $derived (derived_alias_names($host), derived_vhost_alias_names($host)) { push @problems, problem($host, 'redundant-derived-name', "Name $derived is derived from madagascar.xdev.ro") if $declared{$derived}; } push @problems, problem($host, 'missing-ip', 'Host is missing a canonical routable IP') unless canonical_ip($host) || ($host->{status} || '') ne 'active'; } return \@problems; } sub host_payload { my ($host, $tls) = @_; my %copy = %$host; $copy{fqdn} = canonical_host_fqdn($host); $copy{ip} = canonical_ip($host); $copy{names} = [ effective_names($host) ]; $copy{declared_names} = [ declared_dns_names($host) ]; $copy{aliases} = [ declared_alias_names($host) ]; $copy{derived_aliases} = [ derived_alias_names($host) ]; $copy{vhosts} = [ declared_vhost_names($host) ]; $copy{derived_vhost_aliases} = [ derived_vhost_alias_names($host) ]; $copy{certificate_id} = clean_scalar($tls->{certificate_id} || ''); $copy{certificate} = $tls->{certificate} if $tls && ref($tls->{certificate}) eq 'HASH'; return \%copy; } sub effective_names { my ($host) = @_; my @names = declared_dns_names($host); push @names, derived_alias_names($host), derived_vhost_alias_names($host); return unique_preserve(@names); } sub host_dns_names { my ($host) = @_; my @names; my $fqdn = canonical_host_fqdn($host); push @names, $fqdn if length $fqdn; push @names, declared_alias_names($host), derived_alias_names($host); return unique_preserve(@names); } sub vhost_cname_records { my ($host) = @_; my $target = canonical_host_fqdn($host); return () unless length $target; my @records; for my $vhost (declared_vhost_names($host)) { push @records, [ $vhost, $target ]; if (my $short = short_alias_for_fqdn($vhost)) { push @records, [ $short, $target ]; } } my %seen; return grep { !$seen{$_->[0]}++ } @records; } sub declared_dns_names { my ($host) = @_; my @names; my $fqdn = canonical_host_fqdn($host); push @names, $fqdn if length $fqdn; push @names, declared_alias_names($host); push @names, declared_vhost_names($host); return unique_preserve(@names); } sub declared_alias_names { my ($host) = @_; return unique_preserve(map { normalize_dns_name($_) } @{ $host->{aliases} || [] }); } sub declared_vhost_names { my ($host) = @_; return unique_preserve(map { normalize_dns_name($_) } @{ $host->{vhosts} || [] }); } sub declared_dns_names_legacy { my ($host) = @_; return map { normalize_dns_name($_) } @{ $host->{names} || [] }; } sub split_legacy_names { my ($id, $names) = @_; my $fallback = clean_id($id || ''); my (%result) = ( fqdn => '', aliases => [], vhosts => [], ); for my $name (map { normalize_dns_name($_) } @$names) { next unless length $name; if (!$result{fqdn} && $name =~ /\.madagascar\.xdev\.ro\z/ && !name_is_vhost($name)) { $result{fqdn} = $name; next; } if (!$result{fqdn} && $name =~ /\./ && !name_is_vhost($name)) { $result{fqdn} = $name; next; } if (name_is_vhost($name)) { push @{ $result{vhosts} }, $name; } else { push @{ $result{aliases} }, $name; } } $result{fqdn} ||= $fallback ? "$fallback.madagascar.xdev.ro" : ''; $result{aliases} = [ unique_preserve(grep { $_ ne $result{fqdn} } @{ $result{aliases} }) ]; $result{vhosts} = [ unique_preserve(@{ $result{vhosts} }) ]; return \%result; } sub derived_alias_names { my ($host) = @_; my @derived; my $fqdn = canonical_host_fqdn($host); push @derived, short_alias_for_fqdn($fqdn) if length $fqdn; for my $name (declared_alias_names($host)) { push @derived, short_alias_for_fqdn($name); } return unique_preserve(grep { length $_ } @derived); } sub derived_vhost_alias_names { my ($host) = @_; my @derived; for my $name (declared_vhost_names($host)) { push @derived, short_alias_for_fqdn($name); } return unique_preserve(grep { length $_ } @derived); } sub clean_alias_names { my ($payload) = @_; return clean_name_bucket($payload->{aliases}) if defined $payload->{aliases}; my @legacy = remove_derived_names(clean_list($payload->{names})); return grep { !name_is_vhost($_) && $_ ne canonical_host_fqdn({ %$payload, names => \@legacy }) } @legacy; } sub clean_vhost_names { my ($payload) = @_; return clean_name_bucket($payload->{vhosts}) if defined $payload->{vhosts}; my @legacy = remove_derived_names(clean_list($payload->{names})); return grep { name_is_vhost($_) } @legacy; } sub clean_name_bucket { my ($value) = @_; my @names = clean_list($value); return unique_preserve(map { normalize_dns_name($_) } remove_derived_names(@names)); } sub remove_derived_names { my @names = @_; my %derived; for my $name (@names) { next unless $name =~ /^(.+)\.madagascar\.xdev\.ro$/; $derived{$1} = 1; } return grep { !$derived{$_} } @names; } sub unique_preserve { my @values = @_; my %seen; return grep { !$seen{$_}++ } @values; } sub canonical_ip { my ($host) = @_; return '' unless $host && ref($host) eq 'HASH'; for my $key (qw(ip dns_ip hosts_ip)) { my $value = clean_scalar($host->{$key} || ''); return $value if length $value; } return ''; } sub problem { my ($host, $code, $message) = @_; return { host_id => $host->{id}, code => $code, message => $message }; } sub render_local_hosts_tsv { my ($registry) = @_; my $out = "# Local DNS manifest for the madagascar network.\n"; $out .= "# Generated by scripts/host_manager.pl from the runtime SQLite registry.\n"; $out .= "#\n"; $out .= "# Format:\n"; $out .= "# ipname [aliases...]\n"; $out .= "# CNAMEaliastarget\n"; $out .= "#\n"; $out .= "# Priority rule:\n"; $out .= "# - DHCP lease/reservation on 192.168.2.1 is canonical for LAN IP allocation.\n"; $out .= "# - madagascar.json is canonical for cluster roles and service interfaces.\n"; $out .= "# - This file publishes approved local DNS records derived from those sources.\n"; for my $host (sort { $a->{id} cmp $b->{id} } @{ $registry->{hosts} }) { next unless ($host->{status} || 'active') eq 'active'; my $ip = canonical_ip($host); next unless $ip; my @names = host_dns_names($host); next unless @names; $out .= join("\t", $ip, join(' ', @names)) . "\n"; for my $record (vhost_cname_records($host)) { $out .= join("\t", 'CNAME', @$record) . "\n"; } } return $out; } sub render_monitoring { my ($registry) = @_; my @hosts; for my $host (sort { $a->{id} cmp $b->{id} } @{ $registry->{hosts} }) { next unless ($host->{status} || 'active') eq 'active'; next if ($host->{monitoring} || 'pending') eq 'disabled'; my @names = effective_names($host); push @hosts, { id => $host->{id}, primary_name => $names[0], address => canonical_ip($host), aliases => \@names, fqdn => canonical_host_fqdn($host), declared_names => [ declared_dns_names($host) ], aliases_declared => [ declared_alias_names($host) ], aliases_derived => [ derived_alias_names($host) ], vhosts_declared => [ declared_vhost_names($host) ], vhost_aliases_derived => [ derived_vhost_alias_names($host) ], roles => [ @{ $host->{roles} || [] } ], monitoring => $host->{monitoring} || 'pending', notes => $host->{notes} || '', }; } return { version => $registry->{version}, generated_at => iso_now(), source => $opt{db}, hosts => \@hosts, }; } sub debug_database_tables_payload { my $dbh = dbh(); my @tables; my $sth = $dbh->prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name"); $sth->execute; while (my ($name) = $sth->fetchrow_array) { my $quoted = $dbh->quote_identifier($name); my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM $quoted"); push @tables, { name => $name, rows => int($count || 0), }; } return { database => $opt{db}, generated_at => iso_now(), tables => \@tables, counts => { tables => scalar @tables, rows => sum(map { $_->{rows} } @tables), }, }; } sub debug_database_table_payload { my ($table, $limit) = @_; my $dbh = dbh(); $table = clean_scalar($table); return { error => 'missing_table' } unless length $table; return { error => 'invalid_table' } unless debug_table_exists($dbh, $table); $limit = int($limit || 100); $limit = 1 if $limit < 1; $limit = 500 if $limit > 500; my $quoted = $dbh->quote_identifier($table); my $columns = $dbh->selectall_arrayref("PRAGMA table_info($quoted)", { Slice => {} }) || []; my $indexes = $dbh->selectall_arrayref("PRAGMA index_list($quoted)", { Slice => {} }) || []; my @index_details; for my $index (@$indexes) { my $index_name = $index->{name} || ''; next unless length $index_name; my $quoted_index = $dbh->quote_identifier($index_name); my $index_columns = $dbh->selectall_arrayref("PRAGMA index_info($quoted_index)", { Slice => {} }) || []; push @index_details, { name => $index_name, unique => int($index->{unique} || 0), origin => $index->{origin} || '', partial => int($index->{partial} || 0), columns => [ map { $_->{name} || '' } @$index_columns ], }; } my $foreign_keys = $dbh->selectall_arrayref("PRAGMA foreign_key_list($quoted)", { Slice => {} }) || []; my ($row_count) = $dbh->selectrow_array("SELECT COUNT(*) FROM $quoted"); my $rows = $dbh->selectall_arrayref("SELECT * FROM $quoted LIMIT ?", { Slice => {} }, $limit) || []; return { database => $opt{db}, table => $table, generated_at => iso_now(), limit => $limit, row_count => int($row_count || 0), columns => $columns, indexes => \@index_details, foreign_keys => $foreign_keys, rows => $rows, }; } sub debug_database_table_export_payload { my ($table) = @_; my $dbh = dbh(); $table = clean_scalar($table); return { error => 'missing_table' } unless length $table; return { error => 'invalid_table' } unless debug_table_exists($dbh, $table); my $quoted = $dbh->quote_identifier($table); my $columns = $dbh->selectall_arrayref("PRAGMA table_info($quoted)", { Slice => {} }) || []; my @column_names = map { $_->{name} || '' } @$columns; my ($row_count) = $dbh->selectrow_array("SELECT COUNT(*) FROM $quoted"); my $rows = $dbh->selectall_arrayref("SELECT * FROM $quoted", { Slice => {} }) || []; return { database => $opt{db}, table => $table, generated_at => iso_now(), row_count => int($row_count || 0), columns => \@column_names, rows => $rows, }; } sub render_debug_table_csv { my ($export) = @_; my @columns = @{ $export->{columns} || [] }; my @lines = (join(',', map { csv_cell($_) } @columns)); for my $row (@{ $export->{rows} || [] }) { push @lines, join(',', map { csv_cell($row->{$_}) } @columns); } return join("\n", @lines) . "\n"; } sub csv_cell { my ($value) = @_; $value = '' unless defined $value; $value = "$value"; $value =~ s/"/""/g; return qq("$value") if $value =~ /[",\r\n]/; return $value; } sub debug_table_export_filename { my ($table, $extension) = @_; $table = clean_scalar($table || 'table'); $table =~ s/[^A-Za-z0-9_.-]+/-/g; $table = 'table' unless length $table; return "debug-$table.$extension"; } sub debug_table_exists { my ($dbh, $table) = @_; return 0 unless $table =~ /\A[A-Za-z_][A-Za-z0-9_]*\z/; my ($exists) = $dbh->selectrow_array( "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ? AND name NOT LIKE 'sqlite_%'", undef, $table, ); return $exists ? 1 : 0; } sub sum { my $total = 0; $total += $_ || 0 for @_; return $total; } sub ca_script_path { return "$project_dir/scripts/ca_manager.sh"; } sub ca_dir { return $ENV{HOST_MANAGER_CA_DIR} || "$project_dir/var/ca"; } sub ca_cert_path { return ca_dir() . "/certs/ca.cert.pem"; } sub ca_issued_cert_path { my ($name) = @_; die "unsafe certificate name\n" unless $name =~ /\A[A-Za-z0-9_.-]+\z/; return ca_dir() . "/issued/$name.cert.pem"; } sub ca_issued_key_path { my ($name) = @_; die "unsafe certificate name\n" unless $name =~ /\A[A-Za-z0-9_.-]+\z/; return ca_dir() . "/issued/$name.key.pem"; } sub ca_private_key_exists { my ($name) = @_; return 0 unless clean_certificate_id($name || ''); return -f ca_issued_key_path($name) ? 1 : 0; } sub ca_manager_output { my (@args) = @_; my $script = ca_script_path(); die "CA manager script is missing\n" unless -x $script; local $ENV{HOST_MANAGER_CA_DIR} = ca_dir(); open my $fh, '-|', $script, @args or die "Cannot run CA manager\n"; local $/; my $out = <$fh>; close $fh or die "CA manager failed\n"; return $out || ''; } sub ca_manager_json { my ($command) = @_; my $out = ca_manager_output($command); $out ||= $command eq 'list-json' ? '[]' : '{}'; sync_certificates_from_json($out) if $command eq 'list-json'; return $out; } sub sync_certificates_from_json { my ($json) = @_; my $certs = eval { json_decode($json || '[]') }; return if $@ || ref($certs) ne 'ARRAY'; my $dbh = dbh(); my $now = iso_now(); with_transaction($dbh, sub { for my $cert (@$certs) { next unless ref($cert) eq 'HASH'; my $name = clean_id($cert->{name} || $cert->{serial} || $cert->{fingerprint_sha256} || ''); next unless $name; my @dns_names = map { normalize_dns_name($_) } @{ $cert->{dns_names} || [] }; my $host_fqdn = infer_certificate_host_fqdn($dbh, \@dns_names); my $cert_path = ca_issued_cert_path($name); my $csr_path = ca_dir() . "/csr/$name.csr.pem"; my $serial = clean_scalar($cert->{serial} || ''); my $fingerprint = clean_scalar($cert->{fingerprint_sha256} || ''); $dbh->do( 'INSERT INTO certificates (certificate_id, host_fqdn, common_name, subject, issuer, serial, status, not_before, not_after, fingerprint_sha256, cert_path, csr_path, created_at, updated_at, notes) ' . "VALUES (?, ?, ?, ?, ?, ?, 'issued', ?, ?, ?, ?, ?, ?, ?, '') " . 'ON CONFLICT(certificate_id) DO UPDATE SET host_fqdn = excluded.host_fqdn, common_name = excluded.common_name, ' . 'subject = excluded.subject, issuer = excluded.issuer, serial = excluded.serial, status = excluded.status, ' . 'not_before = excluded.not_before, not_after = excluded.not_after, fingerprint_sha256 = excluded.fingerprint_sha256, ' . 'cert_path = excluded.cert_path, csr_path = excluded.csr_path, updated_at = excluded.updated_at', undef, $name, $host_fqdn || undef, $dns_names[0] || '', clean_scalar($cert->{subject} || ''), clean_scalar($cert->{issuer} || ''), length($serial) ? $serial : undef, clean_scalar($cert->{not_before} || ''), clean_scalar($cert->{not_after} || ''), length($fingerprint) ? $fingerprint : undef, $cert_path, $csr_path, $now, $now, ); $dbh->do('DELETE FROM certificate_dns_names WHERE certificate_id = ?', undef, $name); for my $dns_name (@dns_names) { next unless length $dns_name; $dbh->do( 'INSERT OR IGNORE INTO certificate_dns_names (certificate_id, dns_name) VALUES (?, ?)', undef, $name, $dns_name, ); } } }); } sub infer_certificate_host_fqdn { my ($dbh, $dns_names) = @_; for my $name (@$dns_names) { my ($fqdn) = $dbh->selectrow_array('SELECT fqdn FROM hosts WHERE fqdn = ?', undef, $name); return $fqdn if $fqdn; } for my $name (@$dns_names) { my ($fqdn) = $dbh->selectrow_array('SELECT host_fqdn FROM host_aliases WHERE alias_name = ? AND status = ?', undef, $name, 'active'); return $fqdn if $fqdn; ($fqdn) = $dbh->selectrow_array('SELECT host_fqdn FROM vhosts WHERE vhost_fqdn = ? AND status = ?', undef, $name, 'active'); return $fqdn if $fqdn; } return ''; } sub parse_hosts_yaml { my ($text) = @_; my %registry = ( version => 1, updated_at => '', policy => {}, hosts => [], ); my ($section, $current, $list_key); for my $line (split /\n/, $text) { next if $line =~ /^\s*$/ || $line =~ /^\s*#/; if ($line =~ /^version:\s*(\d+)/) { $registry{version} = int($1); } elsif ($line =~ /^updated_at:\s*(.+)$/) { $registry{updated_at} = yaml_unquote($1); } elsif ($line =~ /^policy:\s*$/) { $section = 'policy'; } elsif ($line =~ /^hosts:\s*$/) { $section = 'hosts'; } elsif (($section || '') eq 'policy' && $line =~ /^ ([A-Za-z0-9_]+):\s*(.+)$/) { $registry{policy}{$1} = yaml_unquote($2); } elsif (($section || '') eq 'hosts' && $line =~ /^ - id:\s*(.+)$/) { $current = { id => yaml_unquote($1), fqdn => '', status => 'active', ip => '', aliases => [], vhosts => [], roles => [], sources => [], monitoring => 'pending', notes => '', }; push @{ $registry{hosts} }, $current; $list_key = undef; } elsif ($current && $line =~ /^ ([A-Za-z0-9_]+):\s*$/) { $list_key = $1; $current->{$list_key} ||= []; } elsif ($current && defined $list_key && $line =~ /^ -\s*(.+)$/) { push @{ $current->{$list_key} }, yaml_unquote($1); } elsif ($current && $line =~ /^ ([A-Za-z0-9_]+):\s*(.*)$/) { my $key = $1; my $value = yaml_unquote($2); if ($key eq 'ip') { $current->{ip} = $value; } elsif ($key eq 'dns_ip' || $key eq 'hosts_ip') { $current->{ip} ||= $value; } elsif ($key eq 'fqdn') { $current->{fqdn} = normalize_dns_name($value); } elsif ($key eq 'names') { # ignored here; legacy list is handled after parsing } else { $current->{$key} = $value; } $list_key = undef; } } for my $host (@{ $registry{hosts} }) { my @legacy_names = @{ $host->{names} || [] }; if (@legacy_names) { my $legacy = split_legacy_names($host->{id}, \@legacy_names); $host->{fqdn} ||= $legacy->{fqdn}; $host->{aliases} = $legacy->{aliases} unless @{ $host->{aliases} || [] }; $host->{vhosts} = $legacy->{vhosts} unless @{ $host->{vhosts} || [] }; } delete $host->{names}; $host->{fqdn} ||= canonical_host_fqdn($host); } return \%registry; } sub render_hosts_yaml { my ($registry) = @_; my $out = "version: " . int($registry->{version} || 1) . "\n"; $out .= "updated_at: " . yq($registry->{updated_at} || iso_now()) . "\n"; $out .= "policy:\n"; for my $key (sort keys %{ $registry->{policy} || {} }) { $out .= " $key: " . yq($registry->{policy}{$key}) . "\n"; } $out .= "hosts:\n"; for my $host (sort { $a->{id} cmp $b->{id} } @{ $registry->{hosts} || [] }) { $out .= " - id: " . yq($host->{id}) . "\n"; $out .= " fqdn: " . yq(canonical_host_fqdn($host)) . "\n"; $out .= " status: " . yq($host->{status} || '') . "\n"; $out .= " ip: " . yq(canonical_ip($host)) . "\n"; for my $key (qw(aliases vhosts roles sources)) { $out .= " $key:\n"; for my $value (@{ $host->{$key} || [] }) { $out .= " - " . yq($value) . "\n"; } } $out .= " monitoring: " . yq($host->{monitoring} || 'pending') . "\n"; $out .= " notes: " . yq($host->{notes} || '') . "\n"; } return $out; } sub parse_work_orders_yaml { my ($text) = @_; my %orders = ( version => 1, work_orders => [], ); my ($section, $current, $list_section, $current_action, $current_item); for my $line (split /\n/, $text) { next if $line =~ /^\s*$/ || $line =~ /^\s*#/; if ($line =~ /^version:\s*(\d+)/) { $orders{version} = int($1); } elsif ($line =~ /^work_orders:\s*$/) { $section = 'work_orders'; } elsif (($section || '') eq 'work_orders' && $line =~ /^ - id:\s*(.+)$/) { $current = { id => yaml_unquote($1), status => 'pending', checklist => [], actions => [], }; push @{ $orders{work_orders} }, $current; $list_section = ''; $current_action = undef; $current_item = undef; } elsif ($current && $line =~ /^ checklist:\s*$/) { $list_section = 'checklist'; $current->{checklist} ||= []; } elsif ($current && $list_section eq 'checklist' && $line =~ /^ - id:\s*(.+)$/) { $current_item = { id => yaml_unquote($1), status => 'pending' }; push @{ $current->{checklist} }, $current_item; $current_action = undef; } elsif ($current_item && $list_section eq 'checklist' && $line =~ /^ ([A-Za-z0-9_]+):\s*(.*)$/) { $current_item->{$1} = yaml_unquote($2); } elsif ($current && $line =~ /^ actions:\s*$/) { $list_section = 'actions'; $current->{actions} ||= []; } elsif ($current && $list_section eq 'actions' && $line =~ /^ - type:\s*(.+)$/) { $current_action = { type => yaml_unquote($1) }; push @{ $current->{actions} }, $current_action; $current_item = undef; } elsif ($current_action && $list_section eq 'actions' && $line =~ /^ ([A-Za-z0-9_]+):\s*(.*)$/) { $current_action->{$1} = yaml_unquote($2); } elsif ($current && $line =~ /^ ([A-Za-z0-9_]+):\s*(.*)$/) { $current->{$1} = yaml_unquote($2); $list_section = ''; $current_action = undef; $current_item = undef; } } return \%orders; } sub render_work_orders_yaml { my ($orders) = @_; my $out = "version: " . int($orders->{version} || 1) . "\n"; $out .= "work_orders:\n"; for my $wo (@{ $orders->{work_orders} || [] }) { $out .= " - id: " . yq($wo->{id}) . "\n"; for my $key (qw(status title reason created_at confirmed_at result)) { next unless exists $wo->{$key} && length($wo->{$key} || ''); $out .= " $key: " . yq($wo->{$key}) . "\n"; } $out .= " checklist:\n"; for my $item (@{ $wo->{checklist} || [] }) { $out .= " - id: " . yq($item->{id}) . "\n"; for my $key (qw(text status owner notes updated_at)) { next unless exists $item->{$key} && length($item->{$key} || ''); $out .= " $key: " . yq($item->{$key}) . "\n"; } } $out .= " actions:\n"; for my $action (@{ $wo->{actions} || [] }) { $out .= " - type: " . yq($action->{type}) . "\n"; for my $key (qw(host_id name)) { next unless exists $action->{$key} && length($action->{$key} || ''); $out .= " $key: " . yq($action->{$key}) . "\n"; } } } return $out; } sub request_payload { my ($headers, $body) = @_; my $type = $headers->{'content-type'} || ''; if ($type =~ m{application/json}) { return json_decode($body || '{}'); } return { parse_params($body || '') }; } sub json_bool { my ($value) = @_; return bless \(my $bool = $value ? 1 : 0), 'HostManager::JSONBool'; } sub json_encode { my ($value) = @_; if (!defined $value) { return 'null'; } my $ref = ref($value); if (!$ref) { return $value if $value =~ /\A-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?\z/; return json_string($value); } if ($ref eq 'HostManager::JSONBool') { return $$value ? 'true' : 'false'; } if ($ref eq 'ARRAY') { return '[' . join(',', map { json_encode($_) } @$value) . ']'; } if ($ref eq 'HASH') { return '{' . join(',', map { json_string($_) . ':' . json_encode($value->{$_}) } sort keys %$value) . '}'; } return json_string("$value"); } sub json_string { my ($value) = @_; $value = '' unless defined $value; $value =~ s/\\/\\\\/g; $value =~ s/"/\\"/g; $value =~ s/\n/\\n/g; $value =~ s/\r/\\r/g; $value =~ s/\t/\\t/g; $value =~ s/([\x00-\x1f])/sprintf("\\u%04x", ord($1))/eg; return qq("$value"); } sub json_decode { my ($text) = @_; my $i = 0; my $len = length($text); my ($parse_value, $parse_string, $parse_array, $parse_object, $parse_number, $skip_ws); $skip_ws = sub { $i++ while $i < $len && substr($text, $i, 1) =~ /\s/; }; $parse_string = sub { die "Expected JSON string\n" unless substr($text, $i, 1) eq '"'; $i++; my $out = ''; while ($i < $len) { my $ch = substr($text, $i++, 1); return $out if $ch eq '"'; if ($ch eq "\\") { die "Bad JSON escape\n" if $i >= $len; my $esc = substr($text, $i++, 1); if ($esc eq '"' || $esc eq "\\" || $esc eq '/') { $out .= $esc; } elsif ($esc eq 'b') { $out .= "\b"; } elsif ($esc eq 'f') { $out .= "\f"; } elsif ($esc eq 'n') { $out .= "\n"; } elsif ($esc eq 'r') { $out .= "\r"; } elsif ($esc eq 't') { $out .= "\t"; } elsif ($esc eq 'u') { my $hex = substr($text, $i, 4); die "Bad JSON unicode escape\n" unless $hex =~ /\A[0-9A-Fa-f]{4}\z/; $out .= chr(hex($hex)); $i += 4; } else { die "Bad JSON escape\n"; } } else { $out .= $ch; } } die "Unterminated JSON string\n"; }; $parse_number = sub { my $start = $i; $i++ if substr($text, $i, 1) eq '-'; $i++ while $i < $len && substr($text, $i, 1) =~ /[0-9]/; if ($i < $len && substr($text, $i, 1) eq '.') { $i++; $i++ while $i < $len && substr($text, $i, 1) =~ /[0-9]/; } if ($i < $len && substr($text, $i, 1) =~ /[eE]/) { $i++; $i++ if $i < $len && substr($text, $i, 1) =~ /[+-]/; $i++ while $i < $len && substr($text, $i, 1) =~ /[0-9]/; } return 0 + substr($text, $start, $i - $start); }; $parse_array = sub { die "Expected JSON array\n" unless substr($text, $i, 1) eq '['; $i++; my @out; $skip_ws->(); if ($i < $len && substr($text, $i, 1) eq ']') { $i++; return \@out; } while (1) { push @out, $parse_value->(); $skip_ws->(); my $ch = substr($text, $i++, 1); last if $ch eq ']'; die "Expected JSON array comma\n" unless $ch eq ','; } return \@out; }; $parse_object = sub { die "Expected JSON object\n" unless substr($text, $i, 1) eq '{'; $i++; my %out; $skip_ws->(); if ($i < $len && substr($text, $i, 1) eq '}') { $i++; return \%out; } while (1) { $skip_ws->(); my $key = $parse_string->(); $skip_ws->(); die "Expected JSON object colon\n" unless substr($text, $i++, 1) eq ':'; $out{$key} = $parse_value->(); $skip_ws->(); my $ch = substr($text, $i++, 1); last if $ch eq '}'; die "Expected JSON object comma\n" unless $ch eq ','; } return \%out; }; $parse_value = sub { $skip_ws->(); die "Unexpected end of JSON\n" if $i >= $len; my $ch = substr($text, $i, 1); return $parse_string->() if $ch eq '"'; return $parse_object->() if $ch eq '{'; return $parse_array->() if $ch eq '['; if (substr($text, $i, 4) eq 'true') { $i += 4; return json_bool(1); } if (substr($text, $i, 5) eq 'false') { $i += 5; return json_bool(0); } if (substr($text, $i, 4) eq 'null') { $i += 4; return undef; } return $parse_number->() if $ch =~ /[-0-9]/; die "Unexpected JSON token\n"; }; my $value = $parse_value->(); $skip_ws->(); die "Trailing JSON content\n" if $i != $len; return $value; } sub parse_params { my ($text) = @_; my %out; for my $pair (split /&/, $text) { next unless length $pair; my ($k, $v) = split /=/, $pair, 2; $out{url_decode($k)} = url_decode($v || ''); } return %out; } sub clean_id { my ($value) = @_; $value = lc clean_scalar($value); $value =~ s/[^a-z0-9_.-]+/-/g; $value =~ s/^-+|-+$//g; return $value; } sub clean_certificate_id { my ($value) = @_; $value = clean_scalar($value); return '' unless length $value; return $value =~ /\A[A-Za-z0-9_.-]+\z/ ? $value : ''; } sub clean_ip { my ($value) = @_; $value = clean_scalar($value); return $value if $value =~ /\A(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}\z/; return ''; } sub clean_mac { my ($value) = @_; $value = lc clean_scalar($value); $value =~ s/-/:/g; return $value if $value =~ /\A[0-9a-f]{2}(?::[0-9a-f]{2}){5}\z/; return ''; } sub normalize_dhcp_name { my ($value) = @_; $value = normalize_dns_name($value || ''); $value =~ s/[^a-z0-9_.-]+/-/g; $value =~ s/^-+|-+$//g; return $value; } sub clean_scalar { my ($value) = @_; $value = '' unless defined $value; $value =~ s/[\r\n\t]+/ /g; $value =~ s/^\s+|\s+$//g; return $value; } sub clean_list { my ($value) = @_; return () unless defined $value; my @items = ref($value) eq 'ARRAY' ? @$value : split /[\s,]+/, $value; my @clean; for my $item (@items) { $item = clean_scalar($item); push @clean, $item if length $item; } return @clean; } sub yq { my ($value) = @_; $value = '' unless defined $value; $value =~ s/\\/\\\\/g; $value =~ s/"/\\"/g; return qq("$value"); } sub yaml_unquote { my ($value) = @_; $value = '' unless defined $value; $value =~ s/^\s+|\s+$//g; if ($value =~ /^"(.*)"$/) { $value = $1; $value =~ s/\\"/"/g; $value =~ s/\\\\/\\/g; } return $value; } sub verify_totp { my ($secret, $otp) = @_; return 0 unless $secret && $otp =~ /^\d{6}$/; my $key = eval { base32_decode($secret) }; return 0 if $@ || !length $key; my $counter = int(time() / 30); for my $offset (-1, 0, 1) { return 1 if totp_code($key, $counter + $offset) eq $otp; } return 0; } sub totp_code { my ($key, $counter) = @_; my $msg = pack('NN', int($counter / 4294967296), $counter & 0xffffffff); my $hash = hmac_sha1($msg, $key); my $offset = ord(substr($hash, -1)) & 0x0f; my $bin = unpack('N', substr($hash, $offset, 4)) & 0x7fffffff; return sprintf('%06d', $bin % 1_000_000); } sub base32_decode { my ($text) = @_; $text = uc($text || ''); $text =~ s/[^A-Z2-7]//g; my %map; my @chars = ('A'..'Z', '2'..'7'); @map{@chars} = (0..31); my ($bits, $value, $out) = (0, 0, ''); for my $char (split //, $text) { die "Invalid base32\n" unless exists $map{$char}; $value = ($value << 5) | $map{$char}; $bits += 5; while ($bits >= 8) { $bits -= 8; $out .= chr(($value >> $bits) & 0xff); } } return $out; } sub create_session { my $nonce = random_hex(24); my $expires = int(time() + 8 * 3600); my $sig = hmac_sha256_hex("$nonce:$expires", $session_secret); my $token = "$nonce:$expires:$sig"; $sessions{$token} = $expires; return $token; } sub is_authenticated { my ($headers) = @_; my $token = cookie_value($headers->{'cookie'} || '', 'hm_session'); return 0 unless $token; my ($nonce, $expires, $sig) = split /:/, $token; return 0 unless $nonce && $expires && $sig; return 0 if $expires < time(); return 0 unless hmac_sha256_hex("$nonce:$expires", $session_secret) eq $sig; return exists $sessions{$token}; } sub expire_session { my ($headers) = @_; my $token = cookie_value($headers->{'cookie'} || '', 'hm_session'); delete $sessions{$token} if $token; } sub cookie_value { my ($cookie, $name) = @_; for my $part (split /;\s*/, $cookie) { my ($k, $v) = split /=/, $part, 2; return $v if defined $k && $k eq $name; } return ''; } sub send_json { my ($client, $status, $payload, $extra_headers) = @_; return send_response($client, $status, json_encode($payload), 'application/json; charset=utf-8', $extra_headers); } sub send_json_raw { my ($client, $status, $json_body, $extra_headers) = @_; return send_response($client, $status, $json_body, 'application/json; charset=utf-8', $extra_headers); } sub send_html { my ($client, $status, $html) = @_; return send_response($client, $status, $html, 'text/html; charset=utf-8'); } sub send_text { my ($client, $status, $text) = @_; return send_response($client, $status, $text, 'text/plain; charset=utf-8'); } sub send_download { my ($client, $status, $content, $type, $filename) = @_; return send_response($client, $status, $content, $type, [ qq(Content-Disposition: attachment; filename="$filename") ]); } sub send_file { my ($client, $path, $type, $filename) = @_; return send_json($client, 404, { error => 'missing_file' }) unless -f $path; return send_download($client, 200, read_file($path), $type, $filename); } sub send_response { my ($client, $status, $body, $type, $extra_headers) = @_; my %reason = (200 => 'OK', 400 => 'Bad Request', 401 => 'Unauthorized', 404 => 'Not Found', 409 => 'Conflict', 500 => 'Internal Server Error', 503 => 'Service Unavailable'); $body = '' unless defined $body; print $client "HTTP/1.1 $status " . ($reason{$status} || 'OK') . "\r\n"; print $client "Content-Type: $type\r\n"; print $client "Content-Length: " . length($body) . "\r\n"; print $client "Cache-Control: no-store\r\n"; print $client "$_\r\n" for @{ $extra_headers || [] }; print $client "Connection: close\r\n\r\n"; print $client $body; } sub read_file { my ($path) = @_; open my $fh, '<', $path or die "Cannot read $path: $!"; local $/; return <$fh>; } sub write_file { my ($path, $content) = @_; open my $fh, '>', $path or die "Cannot write $path: $!"; print {$fh} $content; close $fh or die "Cannot close $path: $!"; } sub backup_file { my ($path) = @_; return unless -f $path; my $backup_dir = "$project_dir/backups/host-manager"; make_path($backup_dir) unless -d $backup_dir; my $name = $path; $name =~ s{.*/}{}; my $stamp = strftime('%Y%m%d_%H%M%S', localtime); write_file("$backup_dir/$name.$stamp.bak", read_file($path)); } sub publish_dns_change { my ($registry, $reason) = @_; $reason = clean_scalar($reason || 'registry-change'); backup_file($opt{local_hosts_tsv}); write_file($opt{local_hosts_tsv}, render_local_hosts_tsv($registry)); my $trigger = $opt{dns_publish_trigger} || ''; return { queued => json_bool(0), file => $opt{local_hosts_tsv}, reason => $reason, } unless length $trigger; ensure_parent_dir($trigger); open my $fh, '>>', $trigger or die "Cannot write DNS publish trigger $trigger: $!"; print {$fh} iso_now() . "\t$reason\n"; close $fh or die "Cannot close DNS publish trigger $trigger: $!"; return { queued => json_bool(1), file => $opt{local_hosts_tsv}, trigger => $trigger, reason => $reason, }; } my $db_handle; my $db_seeded = 0; sub dbh { return $db_handle if $db_handle; ensure_parent_dir($opt{db}); $db_handle = DBI->connect( "dbi:SQLite:dbname=$opt{db}", '', '', { RaiseError => 1, PrintError => 0, AutoCommit => 1, sqlite_unicode => 1, }, ) or die "Cannot open SQLite database $opt{db}\n"; $db_handle->do('PRAGMA journal_mode = WAL'); $db_handle->do('PRAGMA foreign_keys = ON'); create_database_schema($db_handle); seed_database($db_handle) unless $db_seeded++; return $db_handle; } sub create_database_schema { my ($dbh) = @_; $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS schema_meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at TEXT NOT NULL ) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS documents ( name TEXT PRIMARY KEY, content TEXT NOT NULL, updated_at TEXT NOT NULL ) SQL $dbh->do( 'INSERT INTO schema_meta (key, value, updated_at) VALUES (?, ?, ?) ' . 'ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at', undef, 'schema_version', '2', iso_now() ); $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS hosts ( fqdn TEXT PRIMARY KEY, legacy_id TEXT NOT NULL UNIQUE, status TEXT NOT NULL DEFAULT 'active', hosts_ip TEXT NOT NULL DEFAULT '', dns_ip TEXT NOT NULL DEFAULT '', monitoring TEXT NOT NULL DEFAULT 'pending', notes TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL ) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS host_aliases ( alias_name TEXT NOT NULL, host_fqdn TEXT NOT NULL, alias_kind TEXT NOT NULL DEFAULT 'declared', status TEXT NOT NULL DEFAULT 'active', is_dns_published INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL, retired_at TEXT, notes TEXT NOT NULL DEFAULT '', PRIMARY KEY (alias_name, host_fqdn), FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE RESTRICT ) SQL $dbh->do(<<'SQL'); CREATE UNIQUE INDEX IF NOT EXISTS idx_host_aliases_active_name ON host_aliases(alias_name) WHERE status = 'active' SQL $dbh->do(<<'SQL'); CREATE INDEX IF NOT EXISTS idx_host_aliases_host_status ON host_aliases(host_fqdn, status) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS host_roles ( host_fqdn TEXT NOT NULL, role TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'active', created_at TEXT NOT NULL, retired_at TEXT, PRIMARY KEY (host_fqdn, role), FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE RESTRICT ) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS host_sources ( host_fqdn TEXT NOT NULL, source TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'active', created_at TEXT NOT NULL, retired_at TEXT, PRIMARY KEY (host_fqdn, source), FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE RESTRICT ) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS host_flags ( host_fqdn TEXT NOT NULL, flag TEXT NOT NULL, value TEXT NOT NULL DEFAULT '1', created_at TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY (host_fqdn, flag), FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE RESTRICT ) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS host_ssh ( host_fqdn TEXT NOT NULL, profile_name TEXT NOT NULL DEFAULT 'default', username TEXT NOT NULL DEFAULT '', port INTEGER NOT NULL DEFAULT 22, identity_file TEXT NOT NULL DEFAULT '', address TEXT NOT NULL DEFAULT '', local_forward_host TEXT NOT NULL DEFAULT '', local_forward_port INTEGER, remote_forward_host TEXT NOT NULL DEFAULT '', remote_forward_port INTEGER, notes TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY (host_fqdn, profile_name), FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE RESTRICT ) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS host_tls ( host_fqdn TEXT PRIMARY KEY, tls_mode TEXT NOT NULL DEFAULT 'local-ca', certificate_id TEXT, notes TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL, FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE CASCADE, FOREIGN KEY (certificate_id) REFERENCES certificates(certificate_id) ON UPDATE CASCADE ON DELETE SET NULL ) SQL $dbh->do(<<'SQL'); CREATE INDEX IF NOT EXISTS idx_host_tls_certificate ON host_tls(certificate_id) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS certificates ( certificate_id TEXT PRIMARY KEY, host_fqdn TEXT, common_name TEXT NOT NULL DEFAULT '', subject TEXT NOT NULL DEFAULT '', issuer TEXT NOT NULL DEFAULT '', serial TEXT UNIQUE, status TEXT NOT NULL DEFAULT 'issued', not_before TEXT NOT NULL DEFAULT '', not_after TEXT NOT NULL DEFAULT '', fingerprint_sha256 TEXT UNIQUE, cert_path TEXT NOT NULL DEFAULT '', csr_path TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL, notes TEXT NOT NULL DEFAULT '', FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE SET NULL ) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS certificate_dns_names ( certificate_id TEXT NOT NULL, dns_name TEXT NOT NULL, PRIMARY KEY (certificate_id, dns_name), FOREIGN KEY (certificate_id) REFERENCES certificates(certificate_id) ON UPDATE CASCADE ON DELETE CASCADE ) SQL $dbh->do(<<'SQL'); CREATE INDEX IF NOT EXISTS idx_certificate_dns_names_dns_name ON certificate_dns_names(dns_name) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS vhosts ( vhost_fqdn TEXT PRIMARY KEY, host_fqdn TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'active', service_name TEXT NOT NULL DEFAULT '', upstream_url TEXT NOT NULL DEFAULT '', tls_mode TEXT NOT NULL DEFAULT 'local-ca', certificate_id TEXT, notes TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL, FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE RESTRICT, FOREIGN KEY (certificate_id) REFERENCES certificates(certificate_id) ON UPDATE CASCADE ON DELETE SET NULL ) SQL $dbh->do(<<'SQL'); CREATE INDEX IF NOT EXISTS idx_vhosts_host_status ON vhosts(host_fqdn, status) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS data_workers ( worker_id TEXT PRIMARY KEY, worker_type TEXT NOT NULL, name TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'active', source TEXT NOT NULL DEFAULT '', last_run_at TEXT, notes TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL ) SQL $dbh->do(<<'SQL'); CREATE INDEX IF NOT EXISTS idx_data_workers_type_status ON data_workers(worker_type, status) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS dhcp_leases ( lease_key TEXT PRIMARY KEY, worker_id TEXT NOT NULL, host_fqdn TEXT, observed_name TEXT NOT NULL DEFAULT '', ip_address TEXT NOT NULL, mac_address TEXT NOT NULL DEFAULT '', lease_state TEXT NOT NULL DEFAULT '', first_seen TEXT NOT NULL, last_seen TEXT NOT NULL, raw TEXT NOT NULL DEFAULT '', FOREIGN KEY (worker_id) REFERENCES data_workers(worker_id) ON UPDATE CASCADE ON DELETE RESTRICT, FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE SET NULL ) SQL $dbh->do('CREATE INDEX IF NOT EXISTS idx_dhcp_leases_ip ON dhcp_leases(ip_address)'); $dbh->do('CREATE INDEX IF NOT EXISTS idx_dhcp_leases_mac ON dhcp_leases(mac_address)'); $dbh->do('CREATE INDEX IF NOT EXISTS idx_dhcp_leases_worker_last_seen ON dhcp_leases(worker_id, last_seen)'); $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS mdns_observations ( observation_key TEXT PRIMARY KEY, worker_id TEXT NOT NULL, host_fqdn TEXT, observed_name TEXT NOT NULL, ip_address TEXT NOT NULL, rr_type TEXT NOT NULL DEFAULT 'A', ttl INTEGER NOT NULL DEFAULT 0, first_seen TEXT NOT NULL, last_seen TEXT NOT NULL, seen_count INTEGER NOT NULL DEFAULT 1, last_peer TEXT NOT NULL DEFAULT '', raw TEXT NOT NULL DEFAULT '', FOREIGN KEY (worker_id) REFERENCES data_workers(worker_id) ON UPDATE CASCADE ON DELETE RESTRICT, FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE SET NULL ) SQL $dbh->do('CREATE INDEX IF NOT EXISTS idx_mdns_observations_name ON mdns_observations(observed_name)'); $dbh->do('CREATE INDEX IF NOT EXISTS idx_mdns_observations_ip ON mdns_observations(ip_address)'); $dbh->do('CREATE INDEX IF NOT EXISTS idx_mdns_observations_worker_last_seen ON mdns_observations(worker_id, last_seen)'); $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS work_orders ( id TEXT PRIMARY KEY, status TEXT NOT NULL DEFAULT 'pending', title TEXT NOT NULL DEFAULT '', reason TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, confirmed_at TEXT NOT NULL DEFAULT '', result TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL ) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS work_order_checklist ( work_order_id TEXT NOT NULL, item_id TEXT NOT NULL, text TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'pending', owner TEXT NOT NULL DEFAULT '', notes TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL DEFAULT '', PRIMARY KEY (work_order_id, item_id), FOREIGN KEY (work_order_id) REFERENCES work_orders(id) ON UPDATE CASCADE ON DELETE CASCADE ) SQL $dbh->do(<<'SQL'); CREATE TABLE IF NOT EXISTS work_order_actions ( work_order_id TEXT NOT NULL, position INTEGER NOT NULL, type TEXT NOT NULL, host_fqdn TEXT, host_legacy_id TEXT NOT NULL DEFAULT '', name TEXT NOT NULL DEFAULT '', payload TEXT NOT NULL DEFAULT '', PRIMARY KEY (work_order_id, position), FOREIGN KEY (work_order_id) REFERENCES work_orders(id) ON UPDATE CASCADE ON DELETE CASCADE, FOREIGN KEY (host_fqdn) REFERENCES hosts(fqdn) ON UPDATE CASCADE ON DELETE SET NULL ) SQL } sub seed_database { my ($dbh) = @_; seed_default_workers($dbh); if (!db_scalar($dbh, 'SELECT COUNT(*) FROM hosts')) { my $registry = parse_hosts_yaml(legacy_document_text($dbh, 'hosts_yaml', $opt{data}, default_hosts_yaml())); normalize_registry_policy($registry); with_transaction($dbh, sub { import_registry_to_db($dbh, $registry, 0); }); } if (!db_scalar($dbh, 'SELECT COUNT(*) FROM work_orders')) { my $orders = parse_work_orders_yaml(legacy_document_text($dbh, 'work_orders_yaml', $opt{work_orders}, default_work_orders_yaml())); with_transaction($dbh, sub { import_work_orders_to_db($dbh, $orders); }); } seed_mdns_observations_from_yaml($dbh); } sub with_transaction { my ($dbh, $code) = @_; return $code->() unless $dbh->{AutoCommit}; $dbh->begin_work; my $ok = eval { $code->(); 1; }; if (!$ok) { my $err = $@ || 'transaction failed'; eval { $dbh->rollback }; die $err; } $dbh->commit; } sub db_scalar { my ($dbh, $sql, @bind) = @_; my ($value) = $dbh->selectrow_array($sql, undef, @bind); return $value || 0; } sub legacy_document_text { my ($dbh, $name, $seed_path, $default_text) = @_; my $row = $dbh->selectrow_hashref('SELECT content FROM documents WHERE name = ?', undef, $name); return $row->{content} if $row && defined $row->{content}; return read_file($seed_path) if -f $seed_path; return $default_text; } sub load_registry_from_db { my $dbh = dbh(); my $registry = { version => 1, updated_at => db_scalar($dbh, 'SELECT value FROM schema_meta WHERE key = ?', 'registry_updated_at') || '', policy => {}, hosts => [], }; my $sth = $dbh->prepare('SELECT * FROM hosts ORDER BY legacy_id'); $sth->execute; while (my $row = $sth->fetchrow_hashref) { my $fqdn = $row->{fqdn}; push @{ $registry->{hosts} }, { id => $row->{legacy_id}, fqdn => $fqdn, status => $row->{status}, ip => canonical_ip($row), aliases => [ active_aliases_for_host($dbh, $fqdn) ], vhosts => [ active_vhosts_for_host($dbh, $fqdn) ], roles => [ active_values_for_host($dbh, 'host_roles', 'role', $fqdn) ], sources => [ active_values_for_host($dbh, 'host_sources', 'source', $fqdn) ], monitoring => $row->{monitoring}, notes => $row->{notes}, }; } return $registry; } sub save_registry_to_db { my ($registry) = @_; my $dbh = dbh(); with_transaction($dbh, sub { import_registry_to_db($dbh, $registry, 1); set_schema_meta($dbh, 'registry_updated_at', $registry->{updated_at} || iso_now()); }); } sub import_registry_to_db { my ($dbh, $registry, $retire_missing) = @_; my %seen; for my $host (@{ $registry->{hosts} || [] }) { my $fqdn = upsert_host_to_db($dbh, $host); $seen{$fqdn} = 1 if $fqdn; } return unless $retire_missing; my $sth = $dbh->prepare('SELECT fqdn FROM hosts WHERE status <> ?'); $sth->execute('retired'); while (my ($fqdn) = $sth->fetchrow_array) { next if $seen{$fqdn}; retire_host_in_db($dbh, $fqdn); } } sub upsert_host_to_db { my ($dbh, $host) = @_; my $now = iso_now(); my $fqdn = canonical_host_fqdn($host); return '' unless $fqdn; my $legacy_id = clean_id($host->{id} || legacy_id_from_fqdn($fqdn)); my $status = clean_scalar($host->{status} || 'active'); my $ip = canonical_ip($host); my $monitoring = clean_scalar($host->{monitoring} || 'pending'); my $notes = clean_scalar($host->{notes} || ''); $dbh->do( 'INSERT INTO hosts (fqdn, legacy_id, status, hosts_ip, dns_ip, monitoring, notes, created_at, updated_at) ' . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ' . 'ON CONFLICT(fqdn) DO UPDATE SET legacy_id = excluded.legacy_id, status = excluded.status, ' . 'hosts_ip = excluded.hosts_ip, dns_ip = excluded.dns_ip, monitoring = excluded.monitoring, ' . 'notes = excluded.notes, updated_at = excluded.updated_at', undef, $fqdn, $legacy_id, $status, $ip, $ip, $monitoring, $notes, $now, $now, ); sync_host_values($dbh, 'host_roles', 'role', $fqdn, [ clean_list($host->{roles}) ]); sync_host_values($dbh, 'host_sources', 'source', $fqdn, [ clean_list($host->{sources}) ]); sync_host_aliases_and_vhosts($dbh, $fqdn, [ declared_alias_names($host) ], [ declared_vhost_names($host) ]); return $fqdn; } sub upsert_host_tls_row { my ($dbh, $host_fqdn, $certificate_id, $now) = @_; $certificate_id = clean_certificate_id($certificate_id || ''); $dbh->do( 'INSERT INTO host_tls (host_fqdn, tls_mode, certificate_id, notes, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?) ' . 'ON CONFLICT(host_fqdn) DO UPDATE SET tls_mode = excluded.tls_mode, certificate_id = excluded.certificate_id, updated_at = excluded.updated_at', undef, $host_fqdn, length($certificate_id) ? 'local-ca' : 'none', length($certificate_id) ? $certificate_id : undef, '', $now, $now, ); } sub sync_host_values { my ($dbh, $table, $column, $fqdn, $values) = @_; my $now = iso_now(); my %active = map { $_ => 1 } @$values; for my $value (@$values) { $dbh->do( "INSERT INTO $table (host_fqdn, $column, status, created_at, retired_at) VALUES (?, ?, 'active', ?, '') " . "ON CONFLICT(host_fqdn, $column) DO UPDATE SET status = 'active', retired_at = ''", undef, $fqdn, $value, $now, ); } my $sth = $dbh->prepare("SELECT $column FROM $table WHERE host_fqdn = ? AND status = 'active'"); $sth->execute($fqdn); while (my ($value) = $sth->fetchrow_array) { next if $active{$value}; $dbh->do("UPDATE $table SET status = 'retired', retired_at = ? WHERE host_fqdn = ? AND $column = ?", undef, $now, $fqdn, $value); } } sub sync_host_aliases_and_vhosts { my ($dbh, $fqdn, $aliases_in, $vhosts_in) = @_; my $now = iso_now(); my (%aliases, %vhosts); if (my $short = short_alias_for_fqdn($fqdn)) { $aliases{$short} = 1; upsert_alias_to_db($dbh, $fqdn, $short, 'derived', $now); } for my $name (@$aliases_in) { $name = normalize_dns_name($name); next unless length $name; next if $name eq $fqdn; $aliases{$name} = 1; upsert_alias_to_db($dbh, $fqdn, $name, 'declared', $now); if (my $short = short_alias_for_fqdn($name)) { $aliases{$short} = 1; upsert_alias_to_db($dbh, $fqdn, $short, 'derived', $now); } } for my $name (@$vhosts_in) { $name = normalize_dns_name($name); next unless length $name; $vhosts{$name} = 1; upsert_vhost_to_db($dbh, $fqdn, $name, $now); if (my $short = short_alias_for_fqdn($name)) { $aliases{$short} = 1; upsert_alias_to_db($dbh, $fqdn, $short, 'derived-vhost', $now); } } retire_missing_names($dbh, 'host_aliases', 'alias_name', $fqdn, \%aliases, $now); retire_missing_names($dbh, 'vhosts', 'vhost_fqdn', $fqdn, \%vhosts, $now); } sub upsert_alias_to_db { my ($dbh, $fqdn, $alias, $kind, $now) = @_; my ($existing_fqdn) = $dbh->selectrow_array( "SELECT host_fqdn FROM host_aliases WHERE alias_name = ? AND status = 'active'", undef, $alias, ); if ($existing_fqdn && $existing_fqdn ne $fqdn) { if ($kind eq 'derived-vhost') { $dbh->do( "UPDATE host_aliases SET status = 'retired', is_dns_published = 0, retired_at = ? WHERE alias_name = ? AND host_fqdn = ? AND status = 'active'", undef, $now, $alias, $existing_fqdn, ); } else { die "alias_conflict: $alias is already active on $existing_fqdn\n"; } } $dbh->do( 'INSERT INTO host_aliases (alias_name, host_fqdn, alias_kind, status, is_dns_published, created_at, retired_at, notes) ' . "VALUES (?, ?, ?, 'active', 1, ?, '', '') " . "ON CONFLICT(alias_name, host_fqdn) DO UPDATE SET alias_kind = excluded.alias_kind, status = 'active', is_dns_published = 1, retired_at = ''", undef, $alias, $fqdn, $kind, $now, ); } sub upsert_vhost_to_db { my ($dbh, $fqdn, $vhost, $now) = @_; my $service = vhost_service_name($vhost); $dbh->do( 'INSERT INTO vhosts (vhost_fqdn, host_fqdn, status, service_name, upstream_url, tls_mode, certificate_id, notes, created_at, updated_at) ' . "VALUES (?, ?, 'active', ?, '', 'local-ca', NULL, '', ?, ?) " . "ON CONFLICT(vhost_fqdn) DO UPDATE SET host_fqdn = excluded.host_fqdn, status = 'active', " . 'service_name = excluded.service_name, updated_at = excluded.updated_at', undef, $vhost, $fqdn, $service, $now, $now, ); } sub retire_missing_names { my ($dbh, $table, $name_column, $fqdn, $active, $now) = @_; my $sth = $dbh->prepare("SELECT $name_column FROM $table WHERE host_fqdn = ? AND status = 'active'"); $sth->execute($fqdn); while (my ($name) = $sth->fetchrow_array) { next if $active->{$name}; if ($table eq 'host_aliases') { $dbh->do( "UPDATE host_aliases SET status = 'retired', is_dns_published = 0, retired_at = ? WHERE host_fqdn = ? AND alias_name = ?", undef, $now, $fqdn, $name, ); } else { $dbh->do( "UPDATE vhosts SET status = 'retired', updated_at = ? WHERE host_fqdn = ? AND vhost_fqdn = ?", undef, $now, $fqdn, $name, ); } } } sub retire_host_in_db { my ($dbh, $fqdn) = @_; my $now = iso_now(); $dbh->do("UPDATE hosts SET status = 'retired', updated_at = ? WHERE fqdn = ?", undef, $now, $fqdn); $dbh->do("UPDATE host_aliases SET status = 'retired', is_dns_published = 0, retired_at = ? WHERE host_fqdn = ? AND status = 'active'", undef, $now, $fqdn); $dbh->do("UPDATE vhosts SET status = 'retired', updated_at = ? WHERE host_fqdn = ? AND status = 'active'", undef, $now, $fqdn); $dbh->do("UPDATE host_roles SET status = 'retired', retired_at = ? WHERE host_fqdn = ? AND status = 'active'", undef, $now, $fqdn); $dbh->do("UPDATE host_sources SET status = 'retired', retired_at = ? WHERE host_fqdn = ? AND status = 'active'", undef, $now, $fqdn); } sub active_aliases_for_host { my ($dbh, $fqdn) = @_; my @names; my $aliases = $dbh->prepare("SELECT alias_name FROM host_aliases WHERE host_fqdn = ? AND status = 'active' AND is_dns_published = 1 AND alias_kind NOT LIKE 'derived%' ORDER BY alias_name"); $aliases->execute($fqdn); while (my ($name) = $aliases->fetchrow_array) { push @names, $name; } return unique_preserve(@names); } sub active_vhosts_for_host { my ($dbh, $fqdn) = @_; my @names; my $vhosts = $dbh->prepare("SELECT vhost_fqdn FROM vhosts WHERE host_fqdn = ? AND status = 'active' ORDER BY vhost_fqdn"); $vhosts->execute($fqdn); while (my ($name) = $vhosts->fetchrow_array) { push @names, $name; } return unique_preserve(@names); } sub active_values_for_host { my ($dbh, $table, $column, $fqdn) = @_; my @values; my $sth = $dbh->prepare("SELECT $column FROM $table WHERE host_fqdn = ? AND status = 'active' ORDER BY $column"); $sth->execute($fqdn); while (my ($value) = $sth->fetchrow_array) { push @values, $value; } return @values; } sub load_work_orders_from_db { my $dbh = dbh(); my $orders = { version => 1, work_orders => [] }; my $sth = $dbh->prepare('SELECT * FROM work_orders ORDER BY id'); $sth->execute; while (my $row = $sth->fetchrow_hashref) { my $wo = { id => $row->{id}, status => $row->{status}, title => $row->{title}, reason => $row->{reason}, created_at => $row->{created_at}, checklist => [], actions => [], }; $wo->{confirmed_at} = $row->{confirmed_at} if length($row->{confirmed_at} || ''); $wo->{result} = $row->{result} if length($row->{result} || ''); my $items = $dbh->prepare('SELECT * FROM work_order_checklist WHERE work_order_id = ? ORDER BY item_id'); $items->execute($row->{id}); while (my $item = $items->fetchrow_hashref) { my %copy = ( id => $item->{item_id}, text => $item->{text}, status => $item->{status}, ); for my $key (qw(owner notes updated_at)) { $copy{$key} = $item->{$key} if length($item->{$key} || ''); } push @{ $wo->{checklist} }, \%copy; } my $actions = $dbh->prepare('SELECT * FROM work_order_actions WHERE work_order_id = ? ORDER BY position'); $actions->execute($row->{id}); while (my $action = $actions->fetchrow_hashref) { my %copy = ( type => $action->{type} ); $copy{host_id} = $action->{host_legacy_id} if length($action->{host_legacy_id} || ''); $copy{name} = $action->{name} if length($action->{name} || ''); push @{ $wo->{actions} }, \%copy; } push @{ $orders->{work_orders} }, $wo; } return $orders; } sub save_work_orders_to_db { my ($orders) = @_; my $dbh = dbh(); with_transaction($dbh, sub { import_work_orders_to_db($dbh, $orders); }); } sub import_work_orders_to_db { my ($dbh, $orders) = @_; my $now = iso_now(); my %seen; for my $wo (@{ $orders->{work_orders} || [] }) { my $id = clean_scalar($wo->{id} || ''); next unless $id; $seen{$id} = 1; $dbh->do( 'INSERT INTO work_orders (id, status, title, reason, created_at, confirmed_at, result, updated_at) ' . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?) ' . 'ON CONFLICT(id) DO UPDATE SET status = excluded.status, title = excluded.title, reason = excluded.reason, ' . 'created_at = excluded.created_at, confirmed_at = excluded.confirmed_at, result = excluded.result, updated_at = excluded.updated_at', undef, $id, clean_scalar($wo->{status} || 'pending'), clean_scalar($wo->{title} || ''), clean_scalar($wo->{reason} || ''), clean_scalar($wo->{created_at} || $now), clean_scalar($wo->{confirmed_at} || ''), clean_scalar($wo->{result} || ''), $now, ); $dbh->do('DELETE FROM work_order_checklist WHERE work_order_id = ?', undef, $id); for my $item (@{ $wo->{checklist} || [] }) { $dbh->do( 'INSERT INTO work_order_checklist (work_order_id, item_id, text, status, owner, notes, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)', undef, $id, clean_scalar($item->{id} || ''), clean_scalar($item->{text} || ''), clean_scalar($item->{status} || 'pending'), clean_scalar($item->{owner} || ''), clean_scalar($item->{notes} || ''), clean_scalar($item->{updated_at} || ''), ); } $dbh->do('DELETE FROM work_order_actions WHERE work_order_id = ?', undef, $id); my $position = 0; for my $action (@{ $wo->{actions} || [] }) { my $legacy_id = clean_id($action->{host_id} || ''); my $host_fqdn = fqdn_for_legacy_id($dbh, $legacy_id); $dbh->do( 'INSERT INTO work_order_actions (work_order_id, position, type, host_fqdn, host_legacy_id, name, payload) VALUES (?, ?, ?, ?, ?, ?, ?)', undef, $id, $position++, clean_scalar($action->{type} || ''), $host_fqdn || undef, $legacy_id, normalize_dns_name($action->{name} || ''), '', ); } } } sub seed_default_workers { my ($dbh) = @_; my $now = iso_now(); my @workers = ( [ 'dhcp-router', 'dhcp', 'Router DHCP leases', 'admin@192.168.2.1', 'DHCP lease/reservation collector source.' ], [ 'mdns-listener', 'mdns', 'mDNS listener', 'var/mdns-observations.yaml', 'mDNS observation collector source.' ], ); for my $worker (@workers) { $dbh->do( 'INSERT INTO data_workers (worker_id, worker_type, name, status, source, last_run_at, notes, created_at, updated_at) ' . "VALUES (?, ?, ?, 'active', ?, NULL, ?, ?, ?) " . 'ON CONFLICT(worker_id) DO UPDATE SET worker_type = excluded.worker_type, name = excluded.name, ' . 'status = excluded.status, source = excluded.source, notes = excluded.notes, updated_at = excluded.updated_at', undef, @$worker, $now, $now, ); } } sub seed_mdns_observations_from_yaml { my ($dbh) = @_; return if db_scalar($dbh, 'SELECT COUNT(*) FROM mdns_observations'); my $path = "$project_dir/var/mdns-observations.yaml"; return unless -f $path; my $db = parse_mdns_observations_yaml(read_file($path)); with_transaction($dbh, sub { for my $observation (@{ $db->{observations} || [] }) { $dbh->do( 'INSERT INTO mdns_observations (observation_key, worker_id, host_fqdn, observed_name, ip_address, rr_type, ttl, first_seen, last_seen, seen_count, last_peer, raw) ' . "VALUES (?, 'mdns-listener', NULL, ?, ?, 'A', ?, ?, ?, ?, ?, '') " . 'ON CONFLICT(observation_key) DO UPDATE SET observed_name = excluded.observed_name, ip_address = excluded.ip_address, ' . 'ttl = excluded.ttl, last_seen = excluded.last_seen, seen_count = excluded.seen_count, last_peer = excluded.last_peer', undef, clean_scalar($observation->{key} || "$observation->{name}|$observation->{ip}"), clean_scalar($observation->{name} || ''), clean_scalar($observation->{ip} || ''), int($observation->{ttl} || 0), clean_scalar($observation->{first_seen} || iso_now()), clean_scalar($observation->{last_seen} || iso_now()), int($observation->{seen_count} || 1), clean_scalar($observation->{last_peer} || ''), ); } }); } sub parse_mdns_observations_yaml { my ($text) = @_; my %db = ( observations => [] ); my ($section, $current); for my $line (split /\n/, $text || '') { next if $line =~ /^\s*$/ || $line =~ /^\s*#/; if ($line =~ /^observations:\s*$/) { $section = 'observations'; } elsif (($section || '') eq 'observations' && $line =~ /^ - key:\s*(.+)$/) { $current = { key => yaml_unquote($1) }; push @{ $db{observations} }, $current; } elsif ($current && $line =~ /^ ([A-Za-z0-9_]+):\s*(.*)$/) { $current->{$1} = yaml_unquote($2); } } return \%db; } sub set_schema_meta { my ($dbh, $key, $value) = @_; $dbh->do( 'INSERT INTO schema_meta (key, value, updated_at) VALUES (?, ?, ?) ' . 'ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at', undef, $key, defined $value ? $value : '', iso_now(), ); } sub fqdn_for_legacy_id { my ($dbh, $legacy_id) = @_; return '' unless length($legacy_id || ''); my ($fqdn) = $dbh->selectrow_array('SELECT fqdn FROM hosts WHERE legacy_id = ?', undef, $legacy_id); return $fqdn || ''; } sub canonical_host_fqdn { my ($host) = @_; my $fqdn = normalize_dns_name($host->{fqdn} || ''); return $fqdn if length $fqdn; my @names = declared_dns_names_legacy($host); for my $name (@names) { return $name if $name =~ /\.madagascar\.xdev\.ro\z/ && !name_is_vhost($name); } for my $name (@names) { return $name if $name =~ /\./ && !name_is_vhost($name); } my $id = clean_id($host->{id} || ''); return $id ? "$id.madagascar.xdev.ro" : ''; } sub legacy_id_from_fqdn { my ($fqdn) = @_; $fqdn = normalize_dns_name($fqdn); $fqdn =~ s/\.madagascar\.xdev\.ro\z//; $fqdn =~ s/\..*\z//; return clean_id($fqdn); } sub normalize_dns_name { my ($name) = @_; $name = lc clean_scalar($name || ''); $name =~ s/\.\z//; return $name; } sub name_is_vhost { my ($name) = @_; $name = normalize_dns_name($name); return $name =~ /\A(?:pmx|pbs|hosts)\./ ? 1 : 0; } sub vhost_name_is_valid { my ($name) = @_; $name = normalize_dns_name($name); return 0 unless length $name; return 0 unless $name eq 'madagascar.xdev.ro' || $name =~ /\.madagascar\.xdev\.ro\z/; return 0 unless length($name) <= 253; for my $label (split /\./, $name) { return 0 unless length($label) >= 1 && length($label) <= 63; return 0 unless $label =~ /\A[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\z/; } return 1; } sub vhost_service_name { my ($name) = @_; $name = normalize_dns_name($name); return $1 if $name =~ /\A([a-z0-9-]+)\./; return ''; } sub short_alias_for_fqdn { my ($name) = @_; $name = normalize_dns_name($name); return $1 if $name =~ /\A(.+)\.madagascar\.xdev\.ro\z/; return ''; } sub normalize_registry_policy { my ($registry) = @_; $registry->{policy} ||= {}; $registry->{policy}{storage_authority} = 'sqlite-relational'; $registry->{policy}{runtime_database} = $opt{db}; } sub default_hosts_yaml { return <<'YAML'; version: 1 updated_at: "" policy: storage_authority: "sqlite-relational" hosts: YAML } sub default_work_orders_yaml { return <<'YAML'; version: 1 work_orders: YAML } sub ensure_parent_dir { my ($path) = @_; my $dir = dirname($path); make_path($dir) unless -d $dir; } sub url_decode { my ($value) = @_; $value = '' unless defined $value; $value =~ tr/+/ /; $value =~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/eg; return $value; } sub random_hex { my ($bytes) = @_; if (open my $fh, '<:raw', '/dev/urandom') { read($fh, my $raw, $bytes); close $fh; return unpack('H*', $raw); } return sha256_hex(rand() . time() . $$); } sub iso_now { return strftime('%Y-%m-%dT%H:%M:%SZ', gmtime); } sub build_info { my %info = ( revision => '', branch => '', built_at => '', deployed_at => '', dirty => '', ); if ($ENV{HOST_MANAGER_BUILD}) { $info{revision} = clean_scalar($ENV{HOST_MANAGER_BUILD}); return \%info; } my $build_file = "$project_dir/BUILD"; if (-f $build_file) { for my $line (split /\n/, read_file($build_file)) { next unless $line =~ /\A([A-Za-z0-9_.-]+)=(.*)\z/; $info{$1} = clean_scalar($2); } return \%info if $info{revision} || $info{built_at}; } my $revision = git_value('rev-parse --short=12 HEAD'); my $branch = git_value('rev-parse --abbrev-ref HEAD'); $info{revision} = $revision if $revision; $info{branch} = $branch if $branch && $branch ne 'HEAD'; return \%info; } sub git_value { my ($args) = @_; return '' unless -d "$project_dir/.git"; open my $fh, '-|', "git -C '$project_dir' $args 2>/dev/null" or return ''; my $value = <$fh> || ''; close $fh; chomp $value; return clean_scalar($value); } sub build_label { my $info = build_info(); my $revision = $info->{revision} || 'unknown'; my $branch = $info->{branch} || ''; $branch = '' if $branch eq 'HEAD'; my $label = $branch ? "$branch $revision" : $revision; $label .= '+dirty' if ($info->{dirty} || '') eq '1'; return $label; } sub build_title { my $info = build_info(); my $label = build_label(); my $stamp = $info->{deployed_at} || $info->{built_at} || ''; return $stamp ? "$label deployed $stamp" : $label; } sub build_revision { my $info = build_info(); return $info->{revision} || 'unknown'; } sub build_details { my $info = build_info(); my %details = ( app => 'Madagascar Local Authority', revision => $info->{revision} || 'unknown', branch => $info->{branch} || '', dirty => ($info->{dirty} || '') eq '1' ? json_bool(1) : json_bool(0), built_at => $info->{built_at} || '', deployed_at => $info->{deployed_at} || '', label => build_label(), title => build_title(), ); return json_encode(\%details); } sub html_escape { my ($value) = @_; $value = '' unless defined $value; $value =~ s/&/&/g; $value =~ s//>/g; $value =~ s/"/"/g; $value =~ s/'/'/g; return $value; } sub app_html { my $build = html_escape(build_revision()); my $build_title = html_escape(build_title()); my $build_details = html_escape(build_details()); my $html = <<'HTML'; Madagascar Local Authority

Madagascar Local Authority

Overview

__HOST_MANAGER_BUILD__
HTML $html =~ s/__HOST_MANAGER_BUILD_TITLE__/$build_title/g; $html =~ s/__HOST_MANAGER_BUILD__/$build/g; $html =~ s/__HOST_MANAGER_BUILD_DETAILS__/$build_details/g; return $html; }