File size: 4,557 Bytes
d9ae24d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | """Evaluate Scale-MAE reconstruction, kNN transfer and GSD sensitivity."""
import json
import argparse
from pathlib import Path
import matplotlib.pyplot as plt
import numpy as np
import yaml
ROOT = Path(__file__).resolve().parents[1]
def knn_predict(train_features, train_labels, test_features, k=3):
train = train_features / np.maximum(np.linalg.norm(train_features, axis=1, keepdims=True), 1e-8)
test = test_features / np.maximum(np.linalg.norm(test_features, axis=1, keepdims=True), 1e-8)
nearest = np.argsort(-(test @ train.T), axis=1)[:, :min(k, len(train))]
return np.asarray([np.bincount(train_labels[row]).argmax() for row in nearest])
def rgb(image):
return np.clip(image[:3].transpose(1, 2, 0), 0, 1)
def main():
parser = argparse.ArgumentParser(description="Evaluate Scale-MAE outputs")
parser.add_argument("--config", default=str(ROOT / "conf/config.yaml"))
args = parser.parse_args()
cfg = yaml.safe_load(Path(args.config).read_text())
source = ROOT / cfg["paths"]["inference_dir"] / "reconstruction.npz"
if not source.exists():
raise FileNotFoundError("Run inference before evaluation")
data = np.load(source)
for key in ("prediction", "target", "test_features", "train_features", "labels", "gsd"):
if key not in data: raise ValueError(f"inference archive missing {key}")
if not np.isfinite(data["prediction"]).all(): raise FloatingPointError("non-finite inference output")
prediction = knn_predict(data["train_features"], data["train_labels"], data["test_features"])
accuracy = float(np.mean(prediction == data["labels"]))
mse = np.mean((data["prediction"] - data["target"]) ** 2, axis=(1, 2, 3))
low_mse = float(np.mean((data["low_prediction"] - data["low_target"]) ** 2))
high_mse = float(np.mean((data["high_prediction"] - data["high_target"]) ** 2))
gsd_values = sorted(np.unique(data["gsd"]).tolist())
gsd_mse = {str(value): float(np.mean(mse[data["gsd"] == value])) for value in gsd_values}
gsd_accuracy = {str(value): float(np.mean(prediction[data["gsd"] == value] == data["labels"][data["gsd"] == value]))
for value in gsd_values}
result = {"reconstruction_mse": float(np.mean(mse)), "low_frequency_mse": low_mse,
"high_frequency_mse": high_mse, "knn_accuracy": accuracy,
"gsd_reconstruction_mse": gsd_mse, "gsd_knn_accuracy": gsd_accuracy,
"data_source": "synthetic", "protocol": cfg["data"]["protocol"]}
output = ROOT / cfg["paths"]["evaluation_dir"]
output.mkdir(parents=True, exist_ok=True)
(output / "metrics.json").write_text(json.dumps(result, indent=2) + "\n")
figure, axes = plt.subplots(2, 3, figsize=(9, 6))
images = ((data["target"][0], "Original"), (data["low_target"][0], "Low target"),
(data["high_target"][0] + 0.5, "High target"), (data["prediction"][0], "Reconstruction"),
(data["low_prediction"][0], "Low prediction"), (data["high_prediction"][0] + 0.5, "High prediction"))
for axis, (image, title) in zip(axes.flat, images):
axis.imshow(rgb(image)); axis.set_title(title); axis.axis("off")
figure.tight_layout(); figure.savefig(output / "bandpass_reconstruction.png", dpi=160); plt.close(figure)
figure, axis = plt.subplots(figsize=(6, 3.5))
axis.plot(gsd_values, [gsd_mse[str(x)] for x in gsd_values], marker="o", color="#1f77b4")
axis.set(xlabel="GSD (m/pixel)", ylabel="Reconstruction MSE", title="Scale-aware Reconstruction")
axis.grid(alpha=0.25)
figure.tight_layout(); figure.savefig(output / "gsd_reconstruction_error.png", dpi=160); plt.close(figure)
figure, axis = plt.subplots(figsize=(6, 3.5))
axis.plot(gsd_values, [gsd_accuracy[str(x)] for x in gsd_values], marker="s", color="#d62728")
axis.set(xlabel="GSD (m/pixel)", ylabel="kNN accuracy", title="Scale-aware Feature Transfer")
axis.set_ylim(0, 1.05); axis.grid(alpha=0.25)
figure.tight_layout(); figure.savefig(output / "gsd_knn_accuracy.png", dpi=160); plt.close(figure)
figure, axis = plt.subplots(figsize=(5, 3))
axis.bar(["Low frequency", "High frequency"], [low_mse, high_mse], color=["#2a9d8f", "#e76f51"])
axis.set(ylabel="MSE", title="Bandpass Reconstruction Error")
figure.tight_layout(); figure.savefig(output / "frequency_error.png", dpi=160); plt.close(figure)
np.save(output / "features.npy", data["test_features"])
print(json.dumps(result, indent=2)); print("evaluation=", output)
if __name__ == "__main__":
main()
|