LocalAuthority / scripts / sync_local_hosts.sh
Newer Older
381 lines | 12.587kb
Xdev Host Manager authored a week ago
1
#!/usr/bin/env bash
2
#
Xdev Host Manager authored a week ago
3
# sync_local_hosts.sh - Sync local madagascar DNS records to jumper and as01.
Xdev Host Manager authored a week ago
4
#
5

            
6
set -euo pipefail
7

            
8
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
9
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
Bogdan Timofte authored 3 days ago
10
HOST_MANAGER="$PROJECT_DIR/scripts/host_manager.pl"
Xdev Host Manager authored a week ago
11
JUMPER_HOST="jumper.madagascar.xdev.ro"
Xdev Host Manager authored a week ago
12
AS01="admin@192.168.2.2"
13
APPLY=false
14
VERIFY=false
15
TARGET="all"
Bogdan Timofte authored 3 days ago
16
SOURCE="jumper"
Xdev Host Manager authored a week ago
17
NEGATIVE_NAME="nohost.madagascar.xdev.ro"
18

            
19
usage() {
20
    cat << EOF
Bogdan Timofte authored 3 days ago
21
Usage: $0 [--apply] [--verify] [--target all|jumper|as01] [--source jumper|local]
Xdev Host Manager authored a week ago
22

            
23
Default mode is dry-run. Use --apply to change remote resolvers.
Bogdan Timofte authored 3 days ago
24
The default manifest source is the runtime SQLite database on jumper.
Xdev Host Manager authored a week ago
25

            
26
Examples:
27
  $0
28
  $0 --apply --verify
29
  $0 --target as01 --apply
Bogdan Timofte authored 3 days ago
30
  $0 --source local
Xdev Host Manager authored a week ago
31
EOF
32
}
33

            
34
log() {
35
    printf '[INFO] %s\n' "$*"
36
}
37

            
38
die() {
39
    printf '[ERROR] %s\n' "$*" >&2
40
    exit 1
41
}
42

            
43
while [[ $# -gt 0 ]]; do
44
    case "$1" in
45
        --apply) APPLY=true; shift ;;
46
        --verify) VERIFY=true; shift ;;
47
        --target) TARGET="${2:-}"; shift 2 ;;
Bogdan Timofte authored 3 days ago
48
        --source) SOURCE="${2:-}"; shift 2 ;;
Xdev Host Manager authored a week ago
49
        -h|--help) usage; exit 0 ;;
50
        *) die "Unknown option: $1" ;;
51
    esac
52
done
53

            
54
case "$TARGET" in
Xdev Host Manager authored a week ago
55
    all|jumper|is-vpn-gw|as01) ;;
Xdev Host Manager authored a week ago
56
    *) die "Invalid target: $TARGET" ;;
57
esac
58

            
Bogdan Timofte authored 3 days ago
59
case "$SOURCE" in
60
    jumper|local) ;;
61
    *) die "Invalid source: $SOURCE" ;;
62
esac
63

            
Xdev Host Manager authored a week ago
64
if [[ "$TARGET" == "is-vpn-gw" ]]; then
65
    log "Target is-vpn-gw is deprecated; use jumper"
66
    TARGET="jumper"
67
fi
68

            
Xdev Host Manager authored a week ago
69
WORK_DIR="$(mktemp -d)"
70
trap 'rm -rf "$WORK_DIR"' EXIT
71

            
Bogdan Timofte authored 3 days ago
72
MANIFEST_FILE="$WORK_DIR/local-hosts.tsv"
Xdev Host Manager authored a week ago
73
HOSTS_ROWS="$WORK_DIR/hosts.rows"
74
CLOAK_ROWS="$WORK_DIR/cloak.rows"
75
NAMES_FILE="$WORK_DIR/names.txt"
76
MIKROTIK_RSC="$WORK_DIR/as01.rsc"
77
VERIFY_ROWS="$WORK_DIR/verify.rows"
Bogdan Timofte authored 3 days ago
78
CNAME_VERIFY_ROWS="$WORK_DIR/cname-verify.rows"
Xdev Host Manager authored a week ago
79

            
Bogdan Timofte authored 3 days ago
80
touch "$HOSTS_ROWS" "$CLOAK_ROWS" "$NAMES_FILE" "$MIKROTIK_RSC" "$VERIFY_ROWS" "$CNAME_VERIFY_ROWS"
Xdev Host Manager authored a week ago
81

            
82
quote_ros() {
83
    printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
84
}
85

            
Bogdan Timofte authored 3 days ago
86
cloak_pattern() {
87
    printf '=%s' "$1"
88
}
89

            
Xdev Host Manager authored a week ago
90
is_local_jumper() {
Xdev Host Manager authored a week ago
91
    local host_name
92
    host_name="$(hostname -f 2>/dev/null || hostname 2>/dev/null || true)"
93
    [[ "$host_name" == "jumper.madagascar.xdev.ro" ]] && return 0
94
    ip addr show 2>/dev/null | grep -q '192\.168\.2\.100' && return 0
95
    return 1
96
}
97

            
Xdev Host Manager authored a week ago
98
run_jumper_payload() {
Xdev Host Manager authored a week ago
99
    local remote_dir="$1"
Xdev Host Manager authored a week ago
100
    if is_local_jumper; then
Xdev Host Manager authored a week ago
101
        REMOTE_DIR="$remote_dir" bash -s
102
    else
Xdev Host Manager authored a week ago
103
        ssh "$JUMPER_HOST" "REMOTE_DIR='$remote_dir' bash -s"
Xdev Host Manager authored a week ago
104
    fi
105
}
106

            
Bogdan Timofte authored 3 days ago
107
fetch_manifest() {
108
    case "$SOURCE" in
109
        local)
110
            perl "$HOST_MANAGER" --print-local-hosts-tsv
111
            ;;
112
        jumper)
113
            if is_local_jumper; then
114
                perl "$HOST_MANAGER" --print-local-hosts-tsv
115
            else
116
                ssh "$JUMPER_HOST" "perl /usr/local/xdev-host-manager/scripts/host_manager.pl --print-local-hosts-tsv"
117
            fi
118
            ;;
119
    esac > "$MANIFEST_FILE"
120
}
121

            
122
fetch_manifest
123

            
Xdev Host Manager authored a week ago
124
while IFS= read -r line || [[ -n "$line" ]]; do
125
    [[ -z "${line//[[:space:]]/}" ]] && continue
126
    [[ "$line" =~ ^[[:space:]]*# ]] && continue
127

            
Bogdan Timofte authored 4 days ago
128
    line="${line//$'\r'/}"
129
    IFS=$'\t' read -r col1 col2 col3 _ <<< "$line"
Bogdan Timofte authored 3 days ago
130
    record_type="$(printf '%s' "$col1" | tr '[:lower:]' '[:upper:]')"
131
    if [[ "$record_type" == "CNAME" ]]; then
132
        name="${col2:-}"
133
        target="${col3:-}"
134
        [[ -n "$name" && -n "$target" ]] || die "Invalid CNAME row: $line"
135

            
136
        printf '%s\n' "$name" >> "$NAMES_FILE"
Bogdan Timofte authored 3 days ago
137
        printf '%-40s %s\n' "$(cloak_pattern "$name")" "$target" >> "$CLOAK_ROWS"
Bogdan Timofte authored 3 days ago
138
        if [[ "$name" == *.xdev.ro ]]; then
139
            printf '%s %s\n' "$name" "$target" >> "$CNAME_VERIFY_ROWS"
140
        fi
141

            
142
        ros_name="$(quote_ros "$name")"
143
        ros_target="$(quote_ros "$target")"
144
        {
145
            printf '/ip dns static remove [find name="%s"]\n' "$ros_name"
146
            printf '/ip dns static add name="%s" type=CNAME cname="%s" comment="xdev-local managed"\n' "$ros_name" "$ros_target"
147
        } >> "$MIKROTIK_RSC"
148
        continue
149
    fi
150

            
Bogdan Timofte authored 4 days ago
151
    if [[ -n "${col3:-}" ]]; then
152
        ip="$col2"
153
        names="$col3"
154
    else
155
        ip="$col1"
156
        names="$col2"
157
    fi
158
    [[ -n "${ip:-}" && -n "${names:-}" ]] || die "Invalid row: $line"
Xdev Host Manager authored a week ago
159

            
Bogdan Timofte authored 4 days ago
160
    printf '%s   %s\n' "$ip" "$names" >> "$HOSTS_ROWS"
Xdev Host Manager authored a week ago
161

            
162
    for name in $names; do
163
        printf '%s\n' "$name" >> "$NAMES_FILE"
Bogdan Timofte authored 3 days ago
164
        printf '%-40s %s\n' "$(cloak_pattern "$name")" "$ip" >> "$CLOAK_ROWS"
Xdev Host Manager authored a week ago
165
        if [[ "$name" == *.xdev.ro ]]; then
Bogdan Timofte authored 4 days ago
166
            printf '%s %s\n' "$name" "$ip" >> "$VERIFY_ROWS"
Xdev Host Manager authored a week ago
167
        fi
168

            
169
        ros_name="$(quote_ros "$name")"
Bogdan Timofte authored 4 days ago
170
        ros_ip="$(quote_ros "$ip")"
Xdev Host Manager authored a week ago
171
        {
172
            printf '/ip dns static remove [find name="%s"]\n' "$ros_name"
173
            printf '/ip dns static add name="%s" type=A address=%s comment="xdev-local managed"\n' "$ros_name" "$ros_ip"
174
        } >> "$MIKROTIK_RSC"
175
    done
Bogdan Timofte authored 3 days ago
176
done < "$MANIFEST_FILE"
Xdev Host Manager authored a week ago
177

            
178
sort -u "$NAMES_FILE" -o "$NAMES_FILE"
179
printf '/ip dns cache flush\n' >> "$MIKROTIK_RSC"
180
printf ':put "xdev local dns sync complete"\n' >> "$MIKROTIK_RSC"
181

            
Xdev Host Manager authored a week ago
182
sync_jumper() {
Xdev Host Manager authored a week ago
183
    if ! $APPLY; then
Xdev Host Manager authored a week ago
184
        log "Dry-run for $JUMPER_HOST: generated /etc/hosts block"
Xdev Host Manager authored a week ago
185
        sed -n '1,80p' "$HOSTS_ROWS"
Xdev Host Manager authored a week ago
186
        log "Dry-run for $JUMPER_HOST: generated cloaking-rules block"
Xdev Host Manager authored a week ago
187
        sed -n '1,120p' "$CLOAK_ROWS"
188
        return
189
    fi
190

            
Xdev Host Manager authored a week ago
191
    log "Syncing $JUMPER_HOST"
Xdev Host Manager authored a week ago
192
    remote_dir="/tmp/xdev-local-dns.$$"
Xdev Host Manager authored a week ago
193
    if is_local_jumper; then
Xdev Host Manager authored a week ago
194
        mkdir -p "$remote_dir"
195
        cp "$HOSTS_ROWS" "$CLOAK_ROWS" "$NAMES_FILE" "$remote_dir/"
196
    else
Xdev Host Manager authored a week ago
197
        ssh "$JUMPER_HOST" "mkdir -p '$remote_dir'"
198
        scp "$HOSTS_ROWS" "$CLOAK_ROWS" "$NAMES_FILE" "$JUMPER_HOST:$remote_dir/" >/dev/null
Xdev Host Manager authored a week ago
199
    fi
Xdev Host Manager authored a week ago
200
    run_jumper_payload "$remote_dir" <<'REMOTE'
Xdev Host Manager authored a week ago
201
set -euo pipefail
202
stamp="$(date +%Y%m%d_%H%M%S)"
203
cp /etc/hosts "/etc/hosts.bak.$stamp"
204
cp /etc/dnscrypt-proxy/cloaking-rules.txt "/etc/dnscrypt-proxy/cloaking-rules.txt.bak.$stamp"
205

            
206
awk '
Bogdan Timofte authored 3 days ago
207
    NR == FNR { names[$1] = 1; names["=" $1] = 1; next }
Xdev Host Manager authored a week ago
208
    /^# BEGIN XDEV LOCAL DNS MANAGED BLOCK/ { in_managed = 1; next }
209
    /^# END XDEV LOCAL DNS MANAGED BLOCK/ { in_managed = 0; next }
210
    in_managed { next }
211
    /^[[:space:]]*#/ || /^[[:space:]]*$/ { print; next }
212
    {
213
        keep = 1
214
        for (i = 2; i <= NF; i++) {
215
            if ($i in names) keep = 0
216
        }
217
        if (keep) print
218
    }
219
' "$REMOTE_DIR/names.txt" /etc/hosts > "$REMOTE_DIR/hosts.new"
220
{
221
    cat "$REMOTE_DIR/hosts.new"
222
    printf "\n# BEGIN XDEV LOCAL DNS MANAGED BLOCK\n"
223
    cat "$REMOTE_DIR/hosts.rows"
224
    printf "# END XDEV LOCAL DNS MANAGED BLOCK\n"
225
} > /etc/hosts
226

            
227
awk '
228
    NR == FNR { names[$1] = 1; next }
229
    /^# BEGIN XDEV LOCAL DNS MANAGED BLOCK/ { in_managed = 1; next }
230
    /^# END XDEV LOCAL DNS MANAGED BLOCK/ { in_managed = 0; next }
231
    in_managed { next }
232
    /^[[:space:]]*#/ || /^[[:space:]]*$/ { print; next }
233
    {
234
        if (!($1 in names)) print
235
    }
236
' "$REMOTE_DIR/names.txt" /etc/dnscrypt-proxy/cloaking-rules.txt > "$REMOTE_DIR/cloak.new"
237
{
238
    cat "$REMOTE_DIR/cloak.new"
239
    printf "\n# BEGIN XDEV LOCAL DNS MANAGED BLOCK\n"
240
    cat "$REMOTE_DIR/cloak.rows"
241
    printf "# END XDEV LOCAL DNS MANAGED BLOCK\n"
242
} > /etc/dnscrypt-proxy/cloaking-rules.txt
243

            
244
resolved_backup=""
245
if ! grep -Eq '^ReadEtcHosts=no$' /etc/systemd/resolved.conf; then
246
    cp /etc/systemd/resolved.conf "/etc/systemd/resolved.conf.bak.$stamp"
247
    resolved_backup=" /etc/systemd/resolved.conf.bak.$stamp"
248
    if grep -Eq '^ReadEtcHosts=' /etc/systemd/resolved.conf; then
249
        sed -i 's/^ReadEtcHosts=.*/ReadEtcHosts=no/' /etc/systemd/resolved.conf
250
    else
251
        printf "\nReadEtcHosts=no\n" >> /etc/systemd/resolved.conf
252
    fi
253
fi
254

            
255
if grep -Eq '^DNSStubListenerExtra=' /etc/systemd/resolved.conf; then
256
    if [[ -z "$resolved_backup" ]]; then
257
        cp /etc/systemd/resolved.conf "/etc/systemd/resolved.conf.bak.$stamp"
258
        resolved_backup=" /etc/systemd/resolved.conf.bak.$stamp"
259
    fi
260
    sed -i 's/^DNSStubListenerExtra=/#DNSStubListenerExtra=/' /etc/systemd/resolved.conf
261
fi
262

            
263
dnscrypt_resolved_conf="/etc/systemd/resolved.conf.d/20-dnscrypt.conf"
264
mkdir -p /etc/systemd/resolved.conf.d
265
if [[ ! -f "$dnscrypt_resolved_conf" ]] || ! grep -Eq '^DNS=127\.0\.0\.1:5300$' "$dnscrypt_resolved_conf"; then
266
    if [[ -f "$dnscrypt_resolved_conf" ]]; then
267
        cp "$dnscrypt_resolved_conf" "$dnscrypt_resolved_conf.bak.$stamp"
268
        resolved_backup="$resolved_backup $dnscrypt_resolved_conf.bak.$stamp"
269
    fi
270
    cat > "$dnscrypt_resolved_conf" <<'EOF'
271
[Resolve]
272
# Send LAN DNS traffic through dnscrypt-proxy so cloaking-rules are authoritative locally.
273
DNS=127.0.0.1:5300
274
FallbackDNS=
275
EOF
276
fi
277

            
278
dnscrypt_conf="/etc/dnscrypt-proxy/dnscrypt-proxy.toml"
279
expected_listen="listen_addresses = ['127.0.0.1:5300', '192.168.2.100:53']"
280
dnscrypt_backup=""
281
if ! grep -Fqx "$expected_listen" "$dnscrypt_conf"; then
282
    cp "$dnscrypt_conf" "$dnscrypt_conf.bak.$stamp"
283
    dnscrypt_backup=" $dnscrypt_conf.bak.$stamp"
284
    sed -i "s|^listen_addresses = .*|$expected_listen|" "$dnscrypt_conf"
285
fi
286

            
287
allowed_names_file="/etc/dnscrypt-proxy/allowed-names.txt"
288
if ! grep -Eq "^allowed_names_file = '/etc/dnscrypt-proxy/allowed-names.txt'$" "$dnscrypt_conf"; then
289
    if [[ -z "$dnscrypt_backup" ]]; then
290
        cp "$dnscrypt_conf" "$dnscrypt_conf.bak.$stamp"
291
        dnscrypt_backup=" $dnscrypt_conf.bak.$stamp"
292
    fi
293
    if grep -Eq '^[#[:space:]]*allowed_names_file =' "$dnscrypt_conf"; then
294
        sed -i "s|^[#[:space:]]*allowed_names_file = .*|allowed_names_file = '/etc/dnscrypt-proxy/allowed-names.txt'|" "$dnscrypt_conf"
295
    else
296
        sed -i "/^\\[allowed_names\\]/a allowed_names_file = '/etc/dnscrypt-proxy/allowed-names.txt'" "$dnscrypt_conf"
297
    fi
298
fi
299

            
300
for allowed_name in google.com www.google.com; do
301
    grep -Fxq "$allowed_name" "$allowed_names_file" || printf "%s\n" "$allowed_name" >> "$allowed_names_file"
302
done
303

            
304
resolvectl flush-caches || true
305
systemctl restart systemd-resolved
306

            
307
systemctl restart dnscrypt-proxy
308
rm -rf "$REMOTE_DIR"
309
printf "backups: /etc/hosts.bak.%s /etc/dnscrypt-proxy/cloaking-rules.txt.bak.%s%s%s\n" "$stamp" "$stamp" "$resolved_backup" "$dnscrypt_backup"
310
REMOTE
311
}
312

            
313
sync_as01() {
314
    if ! $APPLY; then
315
        log "Dry-run for as01: generated RouterOS commands"
316
        sed -n '1,160p' "$MIKROTIK_RSC"
317
        return
318
    fi
319

            
320
    log "Syncing as01"
321
    ssh "$AS01" "/ip dns static remove [find comment=\"xdev-local managed\"]"$'\n'"$(cat "$MIKROTIK_RSC")"
322
}
323

            
324
verify_resolver() {
325
    local resolver="$1"
326
    local failures=0
327

            
328
    log "Verifying resolver $resolver"
329
    while read -r name expected_ip; do
330
        answers="$(dig @"$resolver" "$name" +short || true)"
331
        if ! grep -Fxq "$expected_ip" <<< "$answers"; then
332
            compact_answers="$(paste -s -d ',' - <<< "$answers")"
333
            printf '[FAIL] %s %s expected %s got %s\n' "$resolver" "$name" "$expected_ip" "${compact_answers:-<empty>}" >&2
334
            failures=$((failures + 1))
335
        fi
336
    done < "$VERIFY_ROWS"
337

            
Bogdan Timofte authored 3 days ago
338
    while read -r name expected_target; do
339
        answers="$(dig @"$resolver" "$name" +short || true)"
340
        target_answers="$(dig @"$resolver" "$expected_target" +short || true)"
341
        expected_target_dot="${expected_target%.}."
342
        if ! grep -Fxq "$expected_target" <<< "$answers" \
343
            && ! grep -Fxq "$expected_target_dot" <<< "$answers" \
344
            && ! grep -Fxf <(printf '%s\n' "$target_answers") <<< "$answers" >/dev/null; then
345
            compact_answers="$(paste -s -d ',' - <<< "$answers")"
346
            compact_target_answers="$(paste -s -d ',' - <<< "$target_answers")"
347
            printf '[FAIL] %s %s expected CNAME %s or flattened %s got %s\n' "$resolver" "$name" "$expected_target" "${compact_target_answers:-<empty>}" "${compact_answers:-<empty>}" >&2
348
            failures=$((failures + 1))
349
        fi
350
    done < "$CNAME_VERIFY_ROWS"
351

            
Xdev Host Manager authored a week ago
352
    dns_status="$(dig @"$resolver" "$NEGATIVE_NAME" +noall +comments | awk '/status:/ {print $6}' | tr -d ',' || true)"
353
    if [[ "$dns_status" != "NXDOMAIN" ]]; then
354
        printf '[FAIL] %s %s expected NXDOMAIN got %s\n' "$resolver" "$NEGATIVE_NAME" "${dns_status:-<empty>}" >&2
355
        failures=$((failures + 1))
356
    fi
357

            
358
    [[ "$failures" -eq 0 ]]
359
}
360

            
361
if $APPLY || ! $VERIFY; then
362
    case "$TARGET" in
363
        all)
Xdev Host Manager authored a week ago
364
            sync_jumper
Xdev Host Manager authored a week ago
365
            sync_as01
366
            ;;
Xdev Host Manager authored a week ago
367
        jumper) sync_jumper ;;
Xdev Host Manager authored a week ago
368
        as01) sync_as01 ;;
369
    esac
370
fi
371

            
372
if $VERIFY; then
373
    case "$TARGET" in
374
        all)
375
            verify_resolver 192.168.2.100
376
            verify_resolver 192.168.2.2
377
            ;;
Xdev Host Manager authored a week ago
378
        jumper) verify_resolver 192.168.2.100 ;;
Xdev Host Manager authored a week ago
379
        as01) verify_resolver 192.168.2.2 ;;
380
    esac
381
fi