LocalAuthority / scripts / sync_local_hosts.sh
Newer Older
353 lines | 11.857kb
Xdev Host Manager authored a week ago
1
#!/usr/bin/env bash
2
#
Xdev Host Manager authored a week ago
3
# sync_local_hosts.sh - Sync local madagascar DNS records to jumper and as01.
Xdev Host Manager authored a week ago
4
#
5

            
6
set -euo pipefail
7

            
8
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
9
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
10
CONFIG_FILE="$PROJECT_DIR/config/local-hosts.tsv"
Xdev Host Manager authored a week ago
11
JUMPER_HOST="jumper.madagascar.xdev.ro"
Xdev Host Manager authored a week ago
12
AS01="admin@192.168.2.2"
13
APPLY=false
14
VERIFY=false
15
TARGET="all"
16
NEGATIVE_NAME="nohost.madagascar.xdev.ro"
17

            
18
usage() {
19
    cat << EOF
Xdev Host Manager authored a week ago
20
Usage: $0 [--apply] [--verify] [--target all|jumper|as01] [-c file]
Xdev Host Manager authored a week ago
21

            
22
Default mode is dry-run. Use --apply to change remote resolvers.
23

            
24
Examples:
25
  $0
26
  $0 --apply --verify
27
  $0 --target as01 --apply
28
EOF
29
}
30

            
31
log() {
32
    printf '[INFO] %s\n' "$*"
33
}
34

            
35
die() {
36
    printf '[ERROR] %s\n' "$*" >&2
37
    exit 1
38
}
39

            
40
while [[ $# -gt 0 ]]; do
41
    case "$1" in
42
        --apply) APPLY=true; shift ;;
43
        --verify) VERIFY=true; shift ;;
44
        --target) TARGET="${2:-}"; shift 2 ;;
45
        -c|--config) CONFIG_FILE="${2:-}"; shift 2 ;;
46
        -h|--help) usage; exit 0 ;;
47
        *) die "Unknown option: $1" ;;
48
    esac
49
done
50

            
51
case "$TARGET" in
Xdev Host Manager authored a week ago
52
    all|jumper|is-vpn-gw|as01) ;;
Xdev Host Manager authored a week ago
53
    *) die "Invalid target: $TARGET" ;;
54
esac
55

            
Xdev Host Manager authored a week ago
56
if [[ "$TARGET" == "is-vpn-gw" ]]; then
57
    log "Target is-vpn-gw is deprecated; use jumper"
58
    TARGET="jumper"
59
fi
60

            
Xdev Host Manager authored a week ago
61
[[ -f "$CONFIG_FILE" ]] || die "Missing config file: $CONFIG_FILE"
62

            
63
WORK_DIR="$(mktemp -d)"
64
trap 'rm -rf "$WORK_DIR"' EXIT
65

            
66
HOSTS_ROWS="$WORK_DIR/hosts.rows"
67
CLOAK_ROWS="$WORK_DIR/cloak.rows"
68
NAMES_FILE="$WORK_DIR/names.txt"
69
MIKROTIK_RSC="$WORK_DIR/as01.rsc"
70
VERIFY_ROWS="$WORK_DIR/verify.rows"
Bogdan Timofte authored 3 days ago
71
CNAME_VERIFY_ROWS="$WORK_DIR/cname-verify.rows"
Xdev Host Manager authored a week ago
72

            
Bogdan Timofte authored 3 days ago
73
touch "$HOSTS_ROWS" "$CLOAK_ROWS" "$NAMES_FILE" "$MIKROTIK_RSC" "$VERIFY_ROWS" "$CNAME_VERIFY_ROWS"
Xdev Host Manager authored a week ago
74

            
75
quote_ros() {
76
    printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
77
}
78

            
Xdev Host Manager authored a week ago
79
is_local_jumper() {
Xdev Host Manager authored a week ago
80
    local host_name
81
    host_name="$(hostname -f 2>/dev/null || hostname 2>/dev/null || true)"
82
    [[ "$host_name" == "jumper.madagascar.xdev.ro" ]] && return 0
83
    ip addr show 2>/dev/null | grep -q '192\.168\.2\.100' && return 0
84
    return 1
85
}
86

            
Xdev Host Manager authored a week ago
87
run_jumper_payload() {
Xdev Host Manager authored a week ago
88
    local remote_dir="$1"
Xdev Host Manager authored a week ago
89
    if is_local_jumper; then
Xdev Host Manager authored a week ago
90
        REMOTE_DIR="$remote_dir" bash -s
91
    else
Xdev Host Manager authored a week ago
92
        ssh "$JUMPER_HOST" "REMOTE_DIR='$remote_dir' bash -s"
Xdev Host Manager authored a week ago
93
    fi
94
}
95

            
Xdev Host Manager authored a week ago
96
while IFS= read -r line || [[ -n "$line" ]]; do
97
    [[ -z "${line//[[:space:]]/}" ]] && continue
98
    [[ "$line" =~ ^[[:space:]]*# ]] && continue
99

            
Bogdan Timofte authored 4 days ago
100
    line="${line//$'\r'/}"
101
    IFS=$'\t' read -r col1 col2 col3 _ <<< "$line"
Bogdan Timofte authored 3 days ago
102
    record_type="$(printf '%s' "$col1" | tr '[:lower:]' '[:upper:]')"
103
    if [[ "$record_type" == "CNAME" ]]; then
104
        name="${col2:-}"
105
        target="${col3:-}"
106
        [[ -n "$name" && -n "$target" ]] || die "Invalid CNAME row: $line"
107

            
108
        printf '%s\n' "$name" >> "$NAMES_FILE"
109
        printf '%-40s %s\n' "$name" "$target" >> "$CLOAK_ROWS"
110
        if [[ "$name" == *.xdev.ro ]]; then
111
            printf '%s %s\n' "$name" "$target" >> "$CNAME_VERIFY_ROWS"
112
        fi
113

            
114
        ros_name="$(quote_ros "$name")"
115
        ros_target="$(quote_ros "$target")"
116
        {
117
            printf '/ip dns static remove [find name="%s"]\n' "$ros_name"
118
            printf '/ip dns static add name="%s" type=CNAME cname="%s" comment="xdev-local managed"\n' "$ros_name" "$ros_target"
119
        } >> "$MIKROTIK_RSC"
120
        continue
121
    fi
122

            
Bogdan Timofte authored 4 days ago
123
    if [[ -n "${col3:-}" ]]; then
124
        ip="$col2"
125
        names="$col3"
126
    else
127
        ip="$col1"
128
        names="$col2"
129
    fi
130
    [[ -n "${ip:-}" && -n "${names:-}" ]] || die "Invalid row: $line"
Xdev Host Manager authored a week ago
131

            
Bogdan Timofte authored 4 days ago
132
    printf '%s   %s\n' "$ip" "$names" >> "$HOSTS_ROWS"
Xdev Host Manager authored a week ago
133

            
134
    for name in $names; do
135
        printf '%s\n' "$name" >> "$NAMES_FILE"
Bogdan Timofte authored 4 days ago
136
        printf '%-40s %s\n' "$name" "$ip" >> "$CLOAK_ROWS"
Xdev Host Manager authored a week ago
137
        if [[ "$name" == *.xdev.ro ]]; then
Bogdan Timofte authored 4 days ago
138
            printf '%s %s\n' "$name" "$ip" >> "$VERIFY_ROWS"
Xdev Host Manager authored a week ago
139
        fi
140

            
141
        ros_name="$(quote_ros "$name")"
Bogdan Timofte authored 4 days ago
142
        ros_ip="$(quote_ros "$ip")"
Xdev Host Manager authored a week ago
143
        {
144
            printf '/ip dns static remove [find name="%s"]\n' "$ros_name"
145
            printf '/ip dns static add name="%s" type=A address=%s comment="xdev-local managed"\n' "$ros_name" "$ros_ip"
146
        } >> "$MIKROTIK_RSC"
147
    done
148
done < "$CONFIG_FILE"
149

            
150
sort -u "$NAMES_FILE" -o "$NAMES_FILE"
151
printf '/ip dns cache flush\n' >> "$MIKROTIK_RSC"
152
printf ':put "xdev local dns sync complete"\n' >> "$MIKROTIK_RSC"
153

            
Xdev Host Manager authored a week ago
154
sync_jumper() {
Xdev Host Manager authored a week ago
155
    if ! $APPLY; then
Xdev Host Manager authored a week ago
156
        log "Dry-run for $JUMPER_HOST: generated /etc/hosts block"
Xdev Host Manager authored a week ago
157
        sed -n '1,80p' "$HOSTS_ROWS"
Xdev Host Manager authored a week ago
158
        log "Dry-run for $JUMPER_HOST: generated cloaking-rules block"
Xdev Host Manager authored a week ago
159
        sed -n '1,120p' "$CLOAK_ROWS"
160
        return
161
    fi
162

            
Xdev Host Manager authored a week ago
163
    log "Syncing $JUMPER_HOST"
Xdev Host Manager authored a week ago
164
    remote_dir="/tmp/xdev-local-dns.$$"
Xdev Host Manager authored a week ago
165
    if is_local_jumper; then
Xdev Host Manager authored a week ago
166
        mkdir -p "$remote_dir"
167
        cp "$HOSTS_ROWS" "$CLOAK_ROWS" "$NAMES_FILE" "$remote_dir/"
168
    else
Xdev Host Manager authored a week ago
169
        ssh "$JUMPER_HOST" "mkdir -p '$remote_dir'"
170
        scp "$HOSTS_ROWS" "$CLOAK_ROWS" "$NAMES_FILE" "$JUMPER_HOST:$remote_dir/" >/dev/null
Xdev Host Manager authored a week ago
171
    fi
Xdev Host Manager authored a week ago
172
    run_jumper_payload "$remote_dir" <<'REMOTE'
Xdev Host Manager authored a week ago
173
set -euo pipefail
174
stamp="$(date +%Y%m%d_%H%M%S)"
175
cp /etc/hosts "/etc/hosts.bak.$stamp"
176
cp /etc/dnscrypt-proxy/cloaking-rules.txt "/etc/dnscrypt-proxy/cloaking-rules.txt.bak.$stamp"
177

            
178
awk '
179
    NR == FNR { names[$1] = 1; next }
180
    /^# BEGIN XDEV LOCAL DNS MANAGED BLOCK/ { in_managed = 1; next }
181
    /^# END XDEV LOCAL DNS MANAGED BLOCK/ { in_managed = 0; next }
182
    in_managed { next }
183
    /^[[:space:]]*#/ || /^[[:space:]]*$/ { print; next }
184
    {
185
        keep = 1
186
        for (i = 2; i <= NF; i++) {
187
            if ($i in names) keep = 0
188
        }
189
        if (keep) print
190
    }
191
' "$REMOTE_DIR/names.txt" /etc/hosts > "$REMOTE_DIR/hosts.new"
192
{
193
    cat "$REMOTE_DIR/hosts.new"
194
    printf "\n# BEGIN XDEV LOCAL DNS MANAGED BLOCK\n"
195
    cat "$REMOTE_DIR/hosts.rows"
196
    printf "# END XDEV LOCAL DNS MANAGED BLOCK\n"
197
} > /etc/hosts
198

            
199
awk '
200
    NR == FNR { names[$1] = 1; next }
201
    /^# BEGIN XDEV LOCAL DNS MANAGED BLOCK/ { in_managed = 1; next }
202
    /^# END XDEV LOCAL DNS MANAGED BLOCK/ { in_managed = 0; next }
203
    in_managed { next }
204
    /^[[:space:]]*#/ || /^[[:space:]]*$/ { print; next }
205
    {
206
        if (!($1 in names)) print
207
    }
208
' "$REMOTE_DIR/names.txt" /etc/dnscrypt-proxy/cloaking-rules.txt > "$REMOTE_DIR/cloak.new"
209
{
210
    cat "$REMOTE_DIR/cloak.new"
211
    printf "\n# BEGIN XDEV LOCAL DNS MANAGED BLOCK\n"
212
    cat "$REMOTE_DIR/cloak.rows"
213
    printf "# END XDEV LOCAL DNS MANAGED BLOCK\n"
214
} > /etc/dnscrypt-proxy/cloaking-rules.txt
215

            
216
resolved_backup=""
217
if ! grep -Eq '^ReadEtcHosts=no$' /etc/systemd/resolved.conf; then
218
    cp /etc/systemd/resolved.conf "/etc/systemd/resolved.conf.bak.$stamp"
219
    resolved_backup=" /etc/systemd/resolved.conf.bak.$stamp"
220
    if grep -Eq '^ReadEtcHosts=' /etc/systemd/resolved.conf; then
221
        sed -i 's/^ReadEtcHosts=.*/ReadEtcHosts=no/' /etc/systemd/resolved.conf
222
    else
223
        printf "\nReadEtcHosts=no\n" >> /etc/systemd/resolved.conf
224
    fi
225
fi
226

            
227
if grep -Eq '^DNSStubListenerExtra=' /etc/systemd/resolved.conf; then
228
    if [[ -z "$resolved_backup" ]]; then
229
        cp /etc/systemd/resolved.conf "/etc/systemd/resolved.conf.bak.$stamp"
230
        resolved_backup=" /etc/systemd/resolved.conf.bak.$stamp"
231
    fi
232
    sed -i 's/^DNSStubListenerExtra=/#DNSStubListenerExtra=/' /etc/systemd/resolved.conf
233
fi
234

            
235
dnscrypt_resolved_conf="/etc/systemd/resolved.conf.d/20-dnscrypt.conf"
236
mkdir -p /etc/systemd/resolved.conf.d
237
if [[ ! -f "$dnscrypt_resolved_conf" ]] || ! grep -Eq '^DNS=127\.0\.0\.1:5300$' "$dnscrypt_resolved_conf"; then
238
    if [[ -f "$dnscrypt_resolved_conf" ]]; then
239
        cp "$dnscrypt_resolved_conf" "$dnscrypt_resolved_conf.bak.$stamp"
240
        resolved_backup="$resolved_backup $dnscrypt_resolved_conf.bak.$stamp"
241
    fi
242
    cat > "$dnscrypt_resolved_conf" <<'EOF'
243
[Resolve]
244
# Send LAN DNS traffic through dnscrypt-proxy so cloaking-rules are authoritative locally.
245
DNS=127.0.0.1:5300
246
FallbackDNS=
247
EOF
248
fi
249

            
250
dnscrypt_conf="/etc/dnscrypt-proxy/dnscrypt-proxy.toml"
251
expected_listen="listen_addresses = ['127.0.0.1:5300', '192.168.2.100:53']"
252
dnscrypt_backup=""
253
if ! grep -Fqx "$expected_listen" "$dnscrypt_conf"; then
254
    cp "$dnscrypt_conf" "$dnscrypt_conf.bak.$stamp"
255
    dnscrypt_backup=" $dnscrypt_conf.bak.$stamp"
256
    sed -i "s|^listen_addresses = .*|$expected_listen|" "$dnscrypt_conf"
257
fi
258

            
259
allowed_names_file="/etc/dnscrypt-proxy/allowed-names.txt"
260
if ! grep -Eq "^allowed_names_file = '/etc/dnscrypt-proxy/allowed-names.txt'$" "$dnscrypt_conf"; then
261
    if [[ -z "$dnscrypt_backup" ]]; then
262
        cp "$dnscrypt_conf" "$dnscrypt_conf.bak.$stamp"
263
        dnscrypt_backup=" $dnscrypt_conf.bak.$stamp"
264
    fi
265
    if grep -Eq '^[#[:space:]]*allowed_names_file =' "$dnscrypt_conf"; then
266
        sed -i "s|^[#[:space:]]*allowed_names_file = .*|allowed_names_file = '/etc/dnscrypt-proxy/allowed-names.txt'|" "$dnscrypt_conf"
267
    else
268
        sed -i "/^\\[allowed_names\\]/a allowed_names_file = '/etc/dnscrypt-proxy/allowed-names.txt'" "$dnscrypt_conf"
269
    fi
270
fi
271

            
272
for allowed_name in google.com www.google.com; do
273
    grep -Fxq "$allowed_name" "$allowed_names_file" || printf "%s\n" "$allowed_name" >> "$allowed_names_file"
274
done
275

            
276
resolvectl flush-caches || true
277
systemctl restart systemd-resolved
278

            
279
systemctl restart dnscrypt-proxy
280
rm -rf "$REMOTE_DIR"
281
printf "backups: /etc/hosts.bak.%s /etc/dnscrypt-proxy/cloaking-rules.txt.bak.%s%s%s\n" "$stamp" "$stamp" "$resolved_backup" "$dnscrypt_backup"
282
REMOTE
283
}
284

            
285
sync_as01() {
286
    if ! $APPLY; then
287
        log "Dry-run for as01: generated RouterOS commands"
288
        sed -n '1,160p' "$MIKROTIK_RSC"
289
        return
290
    fi
291

            
292
    log "Syncing as01"
293
    ssh "$AS01" "/ip dns static remove [find comment=\"xdev-local managed\"]"$'\n'"$(cat "$MIKROTIK_RSC")"
294
}
295

            
296
verify_resolver() {
297
    local resolver="$1"
298
    local failures=0
299

            
300
    log "Verifying resolver $resolver"
301
    while read -r name expected_ip; do
302
        answers="$(dig @"$resolver" "$name" +short || true)"
303
        if ! grep -Fxq "$expected_ip" <<< "$answers"; then
304
            compact_answers="$(paste -s -d ',' - <<< "$answers")"
305
            printf '[FAIL] %s %s expected %s got %s\n' "$resolver" "$name" "$expected_ip" "${compact_answers:-<empty>}" >&2
306
            failures=$((failures + 1))
307
        fi
308
    done < "$VERIFY_ROWS"
309

            
Bogdan Timofte authored 3 days ago
310
    while read -r name expected_target; do
311
        answers="$(dig @"$resolver" "$name" +short || true)"
312
        target_answers="$(dig @"$resolver" "$expected_target" +short || true)"
313
        expected_target_dot="${expected_target%.}."
314
        if ! grep -Fxq "$expected_target" <<< "$answers" \
315
            && ! grep -Fxq "$expected_target_dot" <<< "$answers" \
316
            && ! grep -Fxf <(printf '%s\n' "$target_answers") <<< "$answers" >/dev/null; then
317
            compact_answers="$(paste -s -d ',' - <<< "$answers")"
318
            compact_target_answers="$(paste -s -d ',' - <<< "$target_answers")"
319
            printf '[FAIL] %s %s expected CNAME %s or flattened %s got %s\n' "$resolver" "$name" "$expected_target" "${compact_target_answers:-<empty>}" "${compact_answers:-<empty>}" >&2
320
            failures=$((failures + 1))
321
        fi
322
    done < "$CNAME_VERIFY_ROWS"
323

            
Xdev Host Manager authored a week ago
324
    dns_status="$(dig @"$resolver" "$NEGATIVE_NAME" +noall +comments | awk '/status:/ {print $6}' | tr -d ',' || true)"
325
    if [[ "$dns_status" != "NXDOMAIN" ]]; then
326
        printf '[FAIL] %s %s expected NXDOMAIN got %s\n' "$resolver" "$NEGATIVE_NAME" "${dns_status:-<empty>}" >&2
327
        failures=$((failures + 1))
328
    fi
329

            
330
    [[ "$failures" -eq 0 ]]
331
}
332

            
333
if $APPLY || ! $VERIFY; then
334
    case "$TARGET" in
335
        all)
Xdev Host Manager authored a week ago
336
            sync_jumper
Xdev Host Manager authored a week ago
337
            sync_as01
338
            ;;
Xdev Host Manager authored a week ago
339
        jumper) sync_jumper ;;
Xdev Host Manager authored a week ago
340
        as01) sync_as01 ;;
341
    esac
342
fi
343

            
344
if $VERIFY; then
345
    case "$TARGET" in
346
        all)
347
            verify_resolver 192.168.2.100
348
            verify_resolver 192.168.2.2
349
            ;;
Xdev Host Manager authored a week ago
350
        jumper) verify_resolver 192.168.2.100 ;;
Xdev Host Manager authored a week ago
351
        as01) verify_resolver 192.168.2.2 ;;
352
    esac
353
fi