autoSMART: Phase 2 — Archive + On-demand retention... · 73016a1

autoSMART: Phase 2 — Archive + On-demand retention + Collectors update
Browse files

bogdan committed 2 weeks ago

**Phase 2 Finalization:**
- Updated schema-v2.sql: Replace automatic enforce_data_retention() cron with on-demand delete_hdd_data_by_serial(serial, keep_catalog)
- Retention policy: Data deleted ONLY on user request (no automatic cleanup)
- Created rename-to-archive-v1.sql: Safe rename of v1 tables to _archive_v1 (for later)
- Updated SmartCollector.pm: Remove differential storage logic, use atomic insert_collection_event()
- Updated smart-collector-daemon.pl: Same modernization (remove bugs, add atomic insert)
- Created deploy-tapia.sh: Deployment script for tapia node (when ready)
- Validated on database (192.168.2.102): delete_hdd_data_by_serial() function tested, production data verified intact

**Status:** READY FOR PRODUCTION
- Zero data loss from schema v1 → v2 (12.630 events + 315.722 param values)
- Waiting for: tapia installation, then deploy collectors and monitor 1-2 weeks before archiving old tables

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>

main

1 parent 5cf81a9

commit 73016a1

Showing 6 changed files with 241 additions and 73 deletions

+1 -1

projects/autoNAS

View

	@@ -1 +1 @@
1		-Subproject commit ec6d560e9debb952a07887d1d6120ea0cf9fcbd4
	1	+Subproject commit 957b7631b97c0c2fb064ef2ac0789e31b6eb7890

+47 -10

projects/autoSMART/lib/SmartCollector.pm

View

+                         eval {
+                             # Detect/handle HDD migration first
+                             my $hdd_id = $self->_detect_or_create_hdd($drive_info, $smart_data);
+                    -        # Check if we should store this reading using differential storage
+                    -        my $should_store = $self->_should_store_reading($hdd_id, $smart_data);
+                    -        if ($should_store->{store}) {
+                    -            # Insert SMART reading with differential storage information
+                    -            $self->_insert_smart_reading_differential($hdd_id, $drive_info, $smart_data, $should_store);
+                    -            $self->_log("Stored SMART data for HDD ID $hdd_id (Serial: $smart_data->{serial_number}, Type: $should_store->{type})", 2);
+                    +        # SCHEMA v2: Store complete reading (no differential storage)
+                    +        my $event_id = $self->_insert_collection_event($hdd_id, $drive_info, $smart_data);
+                    +        if ($event_id) {
+                    +            $self->_log("Stored SMART data: HDD ID $hdd_id, Event ID $event_id (Serial: $smart_data->{serial_number})", 2);
+                             } else {
+                    -            $self->_log("Skipped unchanged SMART data for HDD ID $hdd_id (Serial: $smart_data->{serial_number})", 3);
+                    +            $self->_log("Failed to store SMART data for HDD ID $hdd_id", 1);
+                         };
+                    +=head2 _insert_collection_event (SCHEMA v2)
+                    +SCHEMA v2: Insert complete SMART reading (no differential storage)
+                    +Calls PostgreSQL function: insert_collection_event()
+                    +=cut
+                    +sub _insert_collection_event {
+                    +    my ($self, $hdd_id, $drive_info, $smart_data) = @_;
+                    +    # Build parameters JSON for database function
+                    +    my $params_json = encode_json($smart_data->{parameters} || {});
+                    +    my $checksum = sha256_hex($params_json . ($smart_data->{temperature} || ''));
+                    +    # Call PostgreSQL function: insert_collection_event(hdd_id, serial, node, ts, temp, ok, checksum, params::JSONB)
+                    +    my $sth = $self->{db_handle}->prepare(q{
+                    +        SELECT insert_collection_event(?, ?, ?, NOW(), ?, ?, ?, ?::jsonb)
+                    +    });
+                    +    eval {
+                    +        $sth->execute(
+                    +            $hdd_id,
+                    +            $smart_data->{serial_number},
+                    +            $self->{node_id},
+                    +            $smart_data->{temperature} || undef,
+                    +            1,  # collection_ok = true
+                    +            $checksum,
+                    +            $params_json
+                    +        );
+                    +    };
+                    +    if ($@) {
+                    +        $self->_log("ERROR inserting collection event: $@", 1);
+                    +        return undef;
+                    +    }
+                    +    my ($event_id) = $sth->fetchrow_array();
+                    +    return $event_id;
+                    +}
+                     =head2 DESTROY
+                     Cleanup database connection

+57 -0

projects/autoSMART/scripts/deploy-tapia.sh

View

+                    +#!/bin/bash
+                    +# autoSMART Schema v2 — Deploy to Tapia (secondary node)
+                    +# Date: 2026-05-20
+                    +# Copies updated collectors and restarts service
+                    +set -e
+                    +TAPIA_IP="192.168.2.93"
+                    +TAPIA_USER="root"
+                    +TARGET_DIR="/opt/autoSMART"
+                    +PROJECT_DIR="$(dirname "$(dirname "$0")")"
+                    +echo "🚀 Deploying autoSMART collectors to tapia ($TAPIA_IP)..."
+                    +echo ""
+                    +# 1. Copy updated scripts
+                    +echo "📦 Copying updated scripts..."
+                    +scp -q "$PROJECT_DIR/scripts/smart-collector-daemon.pl" "$TAPIA_USER@$TAPIA_IP:$TARGET_DIR/scripts/"
+                    +scp -q "$PROJECT_DIR/lib/SmartCollector.pm" "$TAPIA_USER@$TAPIA_IP:$TARGET_DIR/lib/"
+                    +echo "✅ Scripts copied"
+                    +echo ""
+                    +# 2. Stop service on tapia
+                    +echo "🛑 Stopping autosmart service on tapia..."
+                    +ssh "$TAPIA_USER@$TAPIA_IP" "systemctl stop autosmart" || true
+                    +sleep 2
+                    +echo "✅ Service stopped"
+                    +echo ""
+                    +# 3. Restart service
+                    +echo "🚀 Starting autosmart service on tapia..."
+                    +ssh "$TAPIA_USER@$TAPIA_IP" "systemctl start autosmart"
+                    +sleep 3
+                    +echo "✅ Service started"
+                    +echo ""
+                    +# 4. Verify service is running
+                    +echo "🔍 Verifying service status..."
+                    +if ssh "$TAPIA_USER@$TAPIA_IP" "systemctl is-active --quiet autosmart"; then
+                    +    echo "✅ autosmart service is running on tapia"
+                    +else
+                    +    echo "❌ autosmart service failed to start!"
+                    +    exit 1
+                    +fi
+                    +echo ""
+                    +# 5. Check recent logs
+                    +echo "📋 Last 5 log entries from tapia:"
+                    +ssh "$TAPIA_USER@$TAPIA_IP" "tail -5 /var/log/syslog | grep autosmart" || true
+                    +echo ""
+                    +echo "✅ Deploy to tapia complete!"
+                    +echo ""
+                    +echo "Next: Verify data collection:"
+                    +echo "  ssh root@ebony \"psql -h 192.168.2.102 -U postgres -d autosmart\""
+                    +echo "  SELECT node_id, COUNT(*) FROM smart_collection_events GROUP BY node_id;"

+22 -29

projects/autoSMART/scripts/smart-collector-daemon.pl

View

+                         # Get or create HDD inventory entry
+                         my $hdd_id = get_or_create_hdd($dbh, $serial, $model, $device);
+                    -    # Check if we should store this reading
+                    +    # SCHEMA v2: Store complete reading via PostgreSQL function
+                         my $params_json = encode_json(\%smart_params);
+                    -    if (!$force_full && !$config->{node}{store_unchanged}) {
+                    -        # Check for recent identical reading
+                    -        my $sth = $dbh->prepare("
+                    -            SELECT id FROM smart_readings
+                    -            WHERE hdd_id = ? AND parameters_json = ?
+                    -            AND timestamp > NOW() - INTERVAL '1 hour'
+                    -            LIMIT 1
+                    -        ");
+                    -        $sth->execute($hdd_id, $params_json);
+                    -        if ($sth->fetchrow_array()) {
+                    -            log_message("  Skipping unchanged parameters") if $debug;
+                    -            return;
+                    -        }
+                    -    }
+                    -    # Store SMART reading
+                    -    my $reading_type = $force_full ? 'full' : 'differential';
+                    +    my $checksum = sha256_hex($params_json . ($temp || ''));
+                         my $sth = $dbh->prepare("
+                    -        INSERT INTO smart_readings (hdd_id, serial_number, device_path, node_id, timestamp, temperature, parameters_json, reading_type)
+                    -        VALUES (?, ?, ?, ?, NOW(), ?, ?::jsonb, ?)
+                    -        RETURNING id
+                    +        SELECT insert_collection_event(?, ?, ?, NOW(), ?, ?, ?, ?::jsonb)
+                         ");
+                    -    my $reading_id = $dbh->selectrow_array($sth, undef, $hdd_id, $serial, $device, $node_id, $temp || 0, $params_json, $reading_type);
+                    -    log_message("  ✓ SMART reading stored (ID: $reading_id, temp: " . ($temp || 0) . "°C, type: $reading_type)") if $debug;
+                    +    my $event_id = $dbh->selectrow_array(
+                    +        $sth, undef,
+                    +        $hdd_id,
+                    +        $serial,
+                    +        $node_id,
+                    +        $temp || 0,
+                    +        1,  # collection_ok = true
+                    +        $checksum,
+                    +        $params_json
+                    +    );
+                    +    if ($event_id) {
+                    +        log_message("  ✓ SMART event stored (ID: $event_id, temp: " . ($temp || 0) . "°C, params: " . scalar(keys %smart_params) . ")") if $debug;
+                    +    } else {
+                    +        log_message("  ✗ Failed to store SMART event for $serial");
+                    +    }
+                     sub get_or_create_hdd {

+64 -0

projects/autoSMART/sql/rename-to-archive-v1.sql

View

+                    +-- autoSMART — Rename v1 tables to archive
+                    +-- Run on-demand after validating collectors work with schema v2
+                    +-- Date: 2026-05-20
+                    +-- Safe: tables are read-only at this point
+                    +BEGIN;
+                    +-- Rename tables
+                    +ALTER TABLE IF EXISTS smart_readings RENAME TO smart_readings_archive_v1;
+                    +ALTER TABLE IF EXISTS smart_thresholds RENAME TO smart_thresholds_archive_v1;
+                    +ALTER TABLE IF EXISTS alert_history RENAME TO alert_history_archive_v1;
+                    +-- Rename sequences
+                    +ALTER SEQUENCE IF EXISTS smart_readings_id_seq RENAME TO smart_readings_archive_v1_id_seq;
+                    +ALTER SEQUENCE IF EXISTS smart_thresholds_id_seq RENAME TO smart_thresholds_archive_v1_id_seq;
+                    +ALTER SEQUENCE IF EXISTS alert_history_id_seq RENAME TO alert_history_archive_v1_id_seq;
+                    +-- Rename indexes
+                    +ALTER INDEX IF EXISTS smart_readings_pkey RENAME TO smart_readings_archive_v1_pkey;
+                    +ALTER INDEX IF EXISTS idx_smart_readings_hdd_id RENAME TO idx_smart_readings_archive_v1_hdd_id;
+                    +ALTER INDEX IF EXISTS idx_smart_readings_timestamp RENAME TO idx_smart_readings_archive_v1_timestamp;
+                    +ALTER INDEX IF EXISTS idx_smart_readings_serial RENAME TO idx_smart_readings_archive_v1_serial;
+                    +ALTER INDEX IF EXISTS idx_smart_readings_device_path RENAME TO idx_smart_readings_archive_v1_device_path;
+                    +ALTER INDEX IF EXISTS idx_smart_readings_type RENAME TO idx_smart_readings_archive_v1_type;
+                    +ALTER INDEX IF EXISTS idx_smart_readings_checksum RENAME TO idx_smart_readings_archive_v1_checksum;
+                    +ALTER INDEX IF EXISTS idx_smart_readings_previous RENAME TO idx_smart_readings_archive_v1_previous;
+                    +ALTER INDEX IF EXISTS idx_smart_readings_parameters RENAME TO idx_smart_readings_archive_v1_parameters;
+                    +ALTER INDEX IF EXISTS idx_smart_readings_changed_params RENAME TO idx_smart_readings_archive_v1_changed_params;
+                    +ALTER INDEX IF EXISTS smart_thresholds_pkey RENAME TO smart_thresholds_archive_v1_pkey;
+                    +ALTER INDEX IF EXISTS uq_parameter_name RENAME TO uq_archive_v1_parameter_name;
+                    +ALTER INDEX IF EXISTS alert_history_pkey RENAME TO alert_history_archive_v1_pkey;
+                    +ALTER INDEX IF EXISTS idx_alert_history_hdd_id RENAME TO idx_alert_history_archive_v1_hdd_id;
+                    +ALTER INDEX IF EXISTS idx_alert_history_sent_at RENAME TO idx_alert_history_archive_v1_sent_at;
+                    +ALTER INDEX IF EXISTS idx_alert_history_severity RENAME TO idx_alert_history_archive_v1_severity;
+                    +ALTER INDEX IF EXISTS idx_alert_history_serial RENAME TO idx_alert_history_archive_v1_serial;
+                    +-- Rename constraints
+                    +ALTER TABLE smart_readings_archive_v1 RENAME CONSTRAINT smart_readings_hdd_id_fkey TO smart_readings_archive_v1_hdd_id_fkey;
+                    +ALTER TABLE smart_readings_archive_v1 RENAME CONSTRAINT smart_readings_previous_reading_id_fkey TO smart_readings_archive_v1_previous_reading_id_fkey;
+                    +ALTER TABLE alert_history_archive_v1 RENAME CONSTRAINT alert_history_hdd_id_fkey TO alert_history_archive_v1_hdd_id_fkey;
+                    +ALTER TABLE alert_history_archive_v1 RENAME CONSTRAINT alert_history_related_reading_id_fkey TO alert_history_archive_v1_related_reading_id_fkey;
+                    +ALTER TABLE alert_history_archive_v1 RENAME CONSTRAINT alert_history_related_prediction_id_fkey TO alert_history_archive_v1_related_prediction_id_fkey;
+                    +COMMIT;
+                    +-- Summary
+                    +DO $$
+                    +BEGIN
+                    +    RAISE NOTICE '✅ Archive rename complete!';
+                    +    RAISE NOTICE 'Tables renamed:';
+                    +    RAISE NOTICE '  • smart_readings → smart_readings_archive_v1';
+                    +    RAISE NOTICE '  • smart_thresholds → smart_thresholds_archive_v1';
+                    +    RAISE NOTICE '  • alert_history → alert_history_archive_v1';
+                    +    RAISE NOTICE '';
+                    +    RAISE NOTICE 'These tables are now read-only archives.';
+                    +    RAISE NOTICE 'All new data flows through schema v2.';
+                    +    RAISE NOTICE '';
+                    +    RAISE NOTICE 'To delete archives (after 4-6 week validation):';
+                    +    RAISE NOTICE '  DROP TABLE smart_readings_archive_v1 CASCADE;';
+                    +    RAISE NOTICE '  DROP TABLE smart_thresholds_archive_v1 CASCADE;';
+                    +    RAISE NOTICE '  DROP TABLE alert_history_archive_v1 CASCADE;';
+                    +END $$;

+50 -33

projects/autoSMART/sql/schema-v2.sql

View

+                     END;
+                     $$ LANGUAGE plpgsql;
+                    -CREATE OR REPLACE FUNCTION enforce_data_retention(p_retain_months INTEGER DEFAULT 24)
+                    -RETURNS INTEGER AS $$
+                    +-- On-demand data deletion: delete all data for a specific HDD (by serial_number)
+                    +-- RETENTION POLICY: On-demand only (no automatic cleanup)
+                    +-- Called from frontend when user requests to remove a drive from monitoring
+                    +CREATE OR REPLACE FUNCTION delete_hdd_data_by_serial(
+                    +    p_serial_number VARCHAR(100),
+                    +    p_keep_catalog  BOOLEAN DEFAULT true
+                    +) RETURNS TABLE(
+                    +    deleted_values BIGINT,
+                    +    deleted_events BIGINT,
+                    +    deleted_catalog INTEGER
+                    +) AS $$
+                     DECLARE
+                    -    v_cutoff_date   DATE;
+                    -    v_dropped       INTEGER := 0;
+                    -    v_rec           RECORD;
+                    +    v_hdd_id            INTEGER;
+                    +    v_deleted_values    BIGINT := 0;
+                    +    v_deleted_events    BIGINT := 0;
+                    +    v_deleted_catalog   INTEGER := 0;
+                     BEGIN
+                    -    v_cutoff_date := (CURRENT_DATE - (p_retain_months || ' months')::INTERVAL)::DATE;
+                    +    -- Get HDD ID for this serial
+                    +    SELECT id INTO v_hdd_id FROM hdd_inventory WHERE serial_number = p_serial_number;
+                    -    FOR v_rec IN
+                    -        SELECT tablename
+                    -        FROM pg_tables
+                    -        WHERE tablename LIKE 'smart_param_values_%'
+                    -          AND schemaname = 'public'
+                    -    LOOP
+                    -        DECLARE
+                    -            v_year  INTEGER;
+                    -            v_month INTEGER;
+                    -            v_parts TEXT[];
+                    -        BEGIN
+                    -            v_parts := regexp_split_to_array(v_rec.tablename, '_');
+                    -            IF array_length(v_parts, 1) >= 5 THEN
+                    -                v_year  := v_parts[4]::INTEGER;
+                    -                v_month := v_parts[5]::INTEGER;
+                    -                IF make_date(v_year, v_month, 1) < v_cutoff_date THEN
+                    -                    EXECUTE format('DROP TABLE IF EXISTS %I', v_rec.tablename);
+                    -                    v_dropped := v_dropped + 1;
+                    -                    RAISE NOTICE 'Dropped old partition: %', v_rec.tablename;
+                    -                END IF;
+                    -            END IF;
+                    -        END;
+                    -    END LOOP;
+                    +    IF v_hdd_id IS NULL THEN
+                    +        RAISE NOTICE 'Serial % not found in inventory', p_serial_number;
+                    +        RETURN QUERY SELECT 0::BIGINT, 0::BIGINT, 0::INTEGER;
+                    +        RETURN;
+                    +    END IF;
+                    +    -- Delete from smart_param_values (cascade via event_id FK)
+                    +    DELETE FROM smart_param_values spv
+                    +    WHERE event_id IN (
+                    +        SELECT id FROM smart_collection_events
+                    +        WHERE hdd_id = v_hdd_id
+                    +    );
+                    +    GET DIAGNOSTICS v_deleted_values = ROW_COUNT;
+                    +    -- Delete from smart_collection_events
+                         DELETE FROM smart_collection_events
+                    -    WHERE collected_at < v_cutoff_date;
+                    +    WHERE hdd_id = v_hdd_id OR serial_number = p_serial_number;
+                    +    GET DIAGNOSTICS v_deleted_events = ROW_COUNT;
+                    +    -- Optionally delete from hdd_inventory and catalog
+                    +    IF NOT p_keep_catalog THEN
+                    +        DELETE FROM smart_param_catalog spc
+                    +        WHERE NOT EXISTS (
+                    +            SELECT 1 FROM smart_param_values
+                    +            WHERE param_id = spc.id
+                    +        );
+                    +        GET DIAGNOSTICS v_deleted_catalog = ROW_COUNT;
+                    +        DELETE FROM hdd_inventory WHERE id = v_hdd_id;
+                    +    END IF;
+                    +    RAISE NOTICE 'Deleted % param values, % events for serial %',
+                    +        v_deleted_values, v_deleted_events, p_serial_number;
+                    -    RETURN v_dropped;
+                    +    RETURN QUERY SELECT v_deleted_values, v_deleted_events, v_deleted_catalog;
+                     END;
+                     $$ LANGUAGE plpgsql;
+                         RAISE NOTICE '✅ autoSMART Schema v2.0 deployed successfully!';
+                         RAISE NOTICE 'New tables: smart_param_catalog, smart_collection_events, smart_param_values (partitioned)';
+                         RAISE NOTICE 'Views: v_latest_param_values, v_drive_health_summary, v_param_trend, v_cluster_overview, v_smart_readings_compat';
+                    -    RAISE NOTICE 'Functions: upsert_param_catalog, insert_collection_event, create_monthly_partition, enforce_data_retention';
+                    +    RAISE NOTICE 'Functions: upsert_param_catalog, insert_collection_event, create_monthly_partition, delete_hdd_data_by_serial';
+                         RAISE NOTICE 'Next step: Run sql/migrate-v1-to-v2.sql to migrate existing data';
+                     END $$;