|
Bogdan Timofte
authored
3 months ago
|
1
|
#!/usr/bin/perl
|
|
|
2
|
|
|
|
3
|
=head1 NAME
|
|
|
4
|
|
|
|
5
|
test-differential-storage.pl - Test differential SMART storage system
|
|
|
6
|
|
|
|
7
|
=head1 DESCRIPTION
|
|
|
8
|
|
|
|
9
|
This script tests the differential storage implementation by:
|
|
|
10
|
1. Creating test HDD entries
|
|
|
11
|
2. Inserting baseline SMART readings
|
|
|
12
|
3. Inserting identical readings (should be skipped)
|
|
|
13
|
4. Inserting readings with small changes (differential storage)
|
|
|
14
|
5. Inserting readings with critical changes (full storage)
|
|
|
15
|
6. Validating storage efficiency and reconstruction
|
|
|
16
|
|
|
|
17
|
=cut
|
|
|
18
|
|
|
|
19
|
use strict;
|
|
|
20
|
use warnings;
|
|
|
21
|
use FindBin qw($Bin);
|
|
|
22
|
use lib "$Bin/../lib";
|
|
|
23
|
|
|
|
24
|
use DBI;
|
|
|
25
|
use JSON::XS;
|
|
|
26
|
use Data::Dumper;
|
|
|
27
|
use Time::HiRes qw(time);
|
|
|
28
|
use Digest::SHA;
|
|
|
29
|
|
|
|
30
|
# Database configuration
|
|
|
31
|
my $config = {
|
|
|
32
|
db_host => $ENV{AUTOSMART_DB_HOST} || 'localhost',
|
|
|
33
|
db_port => $ENV{AUTOSMART_DB_PORT} || '5432',
|
|
|
34
|
db_name => $ENV{AUTOSMART_DB_NAME} || 'autosmart',
|
|
|
35
|
db_user => $ENV{AUTOSMART_DB_USER} || 'autosmart',
|
|
|
36
|
db_pass => $ENV{AUTOSMART_DB_PASS} || 'smartpassword',
|
|
|
37
|
};
|
|
|
38
|
|
|
|
39
|
print "=== autoSMART Differential Storage Test ===\n\n";
|
|
|
40
|
|
|
|
41
|
# Connect to database
|
|
|
42
|
my $dsn = "DBI:Pg:dbname=$config->{db_name};host=$config->{db_host};port=$config->{db_port}";
|
|
|
43
|
my $dbh = DBI->connect($dsn, $config->{db_user}, $config->{db_pass}, {
|
|
|
44
|
RaiseError => 1,
|
|
|
45
|
AutoCommit => 1,
|
|
|
46
|
PrintError => 0
|
|
|
47
|
}) or die "Failed to connect to database: $DBI::errstr\n";
|
|
|
48
|
|
|
|
49
|
print "✓ Connected to database\n";
|
|
|
50
|
|
|
|
51
|
# Clean up any existing test data
|
|
|
52
|
cleanup_test_data($dbh);
|
|
|
53
|
|
|
|
54
|
# Test 1: Create test HDD
|
|
|
55
|
my $test_hdd_id = create_test_hdd($dbh);
|
|
|
56
|
print "✓ Created test HDD (ID: $test_hdd_id)\n";
|
|
|
57
|
|
|
|
58
|
# Test 2: Insert baseline reading
|
|
|
59
|
my $baseline_reading = {
|
|
|
60
|
parameters => {
|
|
|
61
|
'Reallocated_Sector_Ct' => 0,
|
|
|
62
|
'Spin_Retry_Count' => 0,
|
|
|
63
|
'Current_Pending_Sector' => 0,
|
|
|
64
|
'Power_On_Hours' => 1000,
|
|
|
65
|
'Temperature_Celsius' => 35,
|
|
|
66
|
'Load_Cycle_Count' => 5000
|
|
|
67
|
},
|
|
|
68
|
temperature => 35
|
|
|
69
|
};
|
|
|
70
|
|
|
|
71
|
my $baseline_id = insert_test_reading($dbh, $test_hdd_id, $baseline_reading);
|
|
|
72
|
print "✓ Inserted baseline reading (ID: $baseline_id)\n";
|
|
|
73
|
|
|
|
74
|
# Test 3: Insert identical reading (should be skipped)
|
|
|
75
|
sleep(1);
|
|
|
76
|
my $identical_result = test_should_store($dbh, $test_hdd_id, $baseline_reading);
|
|
|
77
|
print "✓ Identical reading test - Should store: " .
|
|
|
78
|
($identical_result->{should_store} ? "YES" : "NO") .
|
|
|
79
|
" (Type: $identical_result->{reading_type})\n";
|
|
|
80
|
|
|
|
81
|
# Test 4: Insert reading with temperature change only (differential)
|
|
|
82
|
my $temp_change_reading = {
|
|
|
83
|
%$baseline_reading,
|
|
|
84
|
temperature => 38
|
|
|
85
|
};
|
|
|
86
|
$temp_change_reading->{parameters}{Temperature_Celsius} = 38;
|
|
|
87
|
|
|
|
88
|
sleep(1);
|
|
|
89
|
my $temp_result = test_should_store($dbh, $test_hdd_id, $temp_change_reading);
|
|
|
90
|
my $temp_id = insert_test_reading($dbh, $test_hdd_id, $temp_change_reading, $temp_result);
|
|
|
91
|
print "✓ Temperature change reading - Should store: " .
|
|
|
92
|
($temp_result->{should_store} ? "YES" : "NO") .
|
|
|
93
|
" (Type: $temp_result->{reading_type}, ID: $temp_id)\n";
|
|
|
94
|
|
|
|
95
|
# Test 5: Insert reading with critical parameter change (full)
|
|
|
96
|
my $critical_reading = {
|
|
|
97
|
%$baseline_reading,
|
|
|
98
|
temperature => 40
|
|
|
99
|
};
|
|
|
100
|
$critical_reading->{parameters}{Reallocated_Sector_Ct} = 1; # Critical parameter change
|
|
|
101
|
$critical_reading->{parameters}{Temperature_Celsius} = 40;
|
|
|
102
|
|
|
|
103
|
sleep(1);
|
|
|
104
|
my $critical_result = test_should_store($dbh, $test_hdd_id, $critical_reading);
|
|
|
105
|
my $critical_id = insert_test_reading($dbh, $test_hdd_id, $critical_reading, $critical_result);
|
|
|
106
|
print "✓ Critical change reading - Should store: " .
|
|
|
107
|
($critical_result->{should_store} ? "YES" : "NO") .
|
|
|
108
|
" (Type: $critical_result->{reading_type}, ID: $critical_id)\n";
|
|
|
109
|
|
|
|
110
|
# Test 6: Validate reconstruction
|
|
|
111
|
print "\n--- Testing Data Reconstruction ---\n";
|
|
|
112
|
test_reconstruction($dbh, $test_hdd_id);
|
|
|
113
|
|
|
|
114
|
# Test 7: Show storage statistics
|
|
|
115
|
print "\n--- Storage Statistics ---\n";
|
|
|
116
|
show_storage_stats($dbh, $test_hdd_id);
|
|
|
117
|
|
|
|
118
|
print "\n=== Test Complete ===\n";
|
|
|
119
|
|
|
|
120
|
$dbh->disconnect();
|
|
|
121
|
|
|
|
122
|
sub cleanup_test_data {
|
|
|
123
|
my ($dbh) = @_;
|
|
|
124
|
|
|
|
125
|
$dbh->do("DELETE FROM smart_readings WHERE serial_number = 'TEST_SERIAL_001'");
|
|
|
126
|
$dbh->do("DELETE FROM hdd_inventory WHERE serial_number = 'TEST_SERIAL_001'");
|
|
|
127
|
}
|
|
|
128
|
|
|
|
129
|
sub create_test_hdd {
|
|
|
130
|
my ($dbh) = @_;
|
|
|
131
|
|
|
|
132
|
my $sql = q{
|
|
|
133
|
INSERT INTO hdd_inventory
|
|
|
134
|
(serial_number, model_name, firmware, size_gb, manufacturer,
|
|
|
135
|
current_device_path, current_node_id, status)
|
|
|
136
|
VALUES ('TEST_SERIAL_001', 'TEST_MODEL_WD', '1.0', 1000, 'Western Digital',
|
|
|
137
|
'/dev/sdb', 'test-node', 'active')
|
|
|
138
|
RETURNING id
|
|
|
139
|
};
|
|
|
140
|
|
|
|
141
|
my $sth = $dbh->prepare($sql);
|
|
|
142
|
$sth->execute();
|
|
|
143
|
|
|
|
144
|
return $sth->fetchrow_array();
|
|
|
145
|
}
|
|
|
146
|
|
|
|
147
|
sub test_should_store {
|
|
|
148
|
my ($dbh, $hdd_id, $reading) = @_;
|
|
|
149
|
|
|
|
150
|
my $parameters_json = encode_json($reading->{parameters});
|
|
|
151
|
my $checksum = Digest::SHA::sha256_hex($parameters_json . ($reading->{temperature} || ''));
|
|
|
152
|
|
|
|
153
|
my $sth = $dbh->prepare(q{
|
|
|
154
|
SELECT should_store_smart_reading(?, ?, ?, NOW())
|
|
|
155
|
});
|
|
|
156
|
|
|
|
157
|
$sth->execute($hdd_id, $parameters_json, $checksum);
|
|
|
158
|
|
|
|
159
|
return $sth->fetchrow_hashref();
|
|
|
160
|
}
|
|
|
161
|
|
|
|
162
|
sub insert_test_reading {
|
|
|
163
|
my ($dbh, $hdd_id, $reading, $storage_info) = @_;
|
|
|
164
|
|
|
|
165
|
# If no storage info provided, get it
|
|
|
166
|
if (!$storage_info) {
|
|
|
167
|
$storage_info = test_should_store($dbh, $hdd_id, $reading);
|
|
|
168
|
return undef unless $storage_info->{should_store};
|
|
|
169
|
}
|
|
|
170
|
|
|
|
171
|
return undef unless $storage_info->{should_store};
|
|
|
172
|
|
|
|
173
|
# For differential readings, only store changed parameters
|
|
|
174
|
my $parameters_to_store;
|
|
|
175
|
if ($storage_info->{reading_type} eq 'differential' && $storage_info->{changed_parameters}) {
|
|
|
176
|
my $changed_params = decode_json($storage_info->{changed_parameters});
|
|
|
177
|
$parameters_to_store = {};
|
|
|
178
|
|
|
|
179
|
for my $param_name (@$changed_params) {
|
|
|
180
|
$parameters_to_store->{$param_name} = $reading->{parameters}{$param_name};
|
|
|
181
|
}
|
|
|
182
|
} else {
|
|
|
183
|
$parameters_to_store = $reading->{parameters};
|
|
|
184
|
}
|
|
|
185
|
|
|
|
186
|
my $sql = q{
|
|
|
187
|
INSERT INTO smart_readings
|
|
|
188
|
(hdd_id, serial_number, device_path, node_id, timestamp,
|
|
|
189
|
collection_ok, temperature, parameters_json, reading_type,
|
|
|
190
|
changes_detected, changed_parameters, previous_reading_id, checksum)
|
|
|
191
|
VALUES (?, ?, ?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?)
|
|
|
192
|
RETURNING id
|
|
|
193
|
};
|
|
|
194
|
|
|
|
195
|
my $parameters_json = encode_json($parameters_to_store);
|
|
|
196
|
my $checksum = Digest::SHA::sha256_hex(encode_json($reading->{parameters}) . ($reading->{temperature} || ''));
|
|
|
197
|
|
|
|
198
|
my $sth = $dbh->prepare($sql);
|
|
|
199
|
$sth->execute(
|
|
|
200
|
$hdd_id,
|
|
|
201
|
'TEST_SERIAL_001',
|
|
|
202
|
'/dev/sdb',
|
|
|
203
|
'test-node',
|
|
|
204
|
1, # collection_ok
|
|
|
205
|
$reading->{temperature},
|
|
|
206
|
$parameters_json,
|
|
|
207
|
$storage_info->{reading_type},
|
|
|
208
|
$storage_info->{changes_detected} ? 1 : 0,
|
|
|
209
|
$storage_info->{changed_parameters},
|
|
|
210
|
$storage_info->{previous_reading_id},
|
|
|
211
|
$checksum
|
|
|
212
|
);
|
|
|
213
|
|
|
|
214
|
return $sth->fetchrow_array();
|
|
|
215
|
}
|
|
|
216
|
|
|
|
217
|
sub test_reconstruction {
|
|
|
218
|
my ($dbh, $hdd_id) = @_;
|
|
|
219
|
|
|
|
220
|
my $sql = q{
|
|
|
221
|
SELECT id, timestamp, reading_type, chain_level, parameters_json, temperature
|
|
|
222
|
FROM smart_readings_reconstructed
|
|
|
223
|
WHERE hdd_id = ?
|
|
|
224
|
ORDER BY timestamp
|
|
|
225
|
};
|
|
|
226
|
|
|
|
227
|
my $sth = $dbh->prepare($sql);
|
|
|
228
|
$sth->execute($hdd_id);
|
|
|
229
|
|
|
|
230
|
while (my $row = $sth->fetchrow_hashref()) {
|
|
|
231
|
print "Reading ID: $row->{id}, Type: $row->{reading_type}, Chain: $row->{chain_level}\n";
|
|
|
232
|
print " Temperature: $row->{temperature}°C\n";
|
|
|
233
|
|
|
|
234
|
my $params = decode_json($row->{parameters_json});
|
|
|
235
|
for my $param (sort keys %$params) {
|
|
|
236
|
print " $param: $params->{$param}\n";
|
|
|
237
|
}
|
|
|
238
|
print "\n";
|
|
|
239
|
}
|
|
|
240
|
}
|
|
|
241
|
|
|
|
242
|
sub show_storage_stats {
|
|
|
243
|
my ($dbh, $hdd_id) = @_;
|
|
|
244
|
|
|
|
245
|
my $sql = q{
|
|
|
246
|
SELECT
|
|
|
247
|
reading_type,
|
|
|
248
|
COUNT(*) as count,
|
|
|
249
|
AVG(length(parameters_json::text)) as avg_size
|
|
|
250
|
FROM smart_readings
|
|
|
251
|
WHERE hdd_id = ?
|
|
|
252
|
GROUP BY reading_type
|
|
|
253
|
ORDER BY reading_type
|
|
|
254
|
};
|
|
|
255
|
|
|
|
256
|
my $sth = $dbh->prepare($sql);
|
|
|
257
|
$sth->execute($hdd_id);
|
|
|
258
|
|
|
|
259
|
my $total_readings = 0;
|
|
|
260
|
my $total_size = 0;
|
|
|
261
|
|
|
|
262
|
while (my $row = $sth->fetchrow_hashref()) {
|
|
|
263
|
printf "%-12s: %d readings, avg size: %.0f bytes\n",
|
|
|
264
|
$row->{reading_type}, $row->{count}, $row->{avg_size};
|
|
|
265
|
$total_readings += $row->{count};
|
|
|
266
|
$total_size += $row->{count} * $row->{avg_size};
|
|
|
267
|
}
|
|
|
268
|
|
|
|
269
|
print "\nTotal: $total_readings readings, estimated size: " . int($total_size) . " bytes\n";
|
|
|
270
|
}
|