From 3151f393bd794fd0798d1f4f7e690c2032ab7ab0 Mon Sep 17 00:00:00 2001 From: Favyen Bastani Date: Thu, 30 Jan 2025 23:52:56 -0500 Subject: [PATCH] Miscellaneous config changes. - Use 32 workers when materializing vessel detection crops. - Just use Jupiter for Sentinel-2 job launcher. - Remove redundant code in Sentinel-2 vessel detection pipeline that could hide an error. - Ecosystem mapping: update to use latest dataset version - Minderoo: removed unneeded inference config and update training config to new config format --- .../config_sentinel2.yaml | 4 +- .../minderoo/classify/config.yaml | 19 ++-- .../minderoo/classify/config_infer.yaml | 95 ------------------- rslp/landsat_vessels/predict_pipeline.py | 2 +- rslp/sentinel2_vessels/job_launcher.py | 3 - rslp/sentinel2_vessels/predict_pipeline.py | 5 +- 6 files changed, 17 insertions(+), 111 deletions(-) delete mode 100644 one_off_projects/minderoo/classify/config_infer.yaml diff --git a/data/maldives_ecosystem_mapping/config_sentinel2.yaml b/data/maldives_ecosystem_mapping/config_sentinel2.yaml index bdcd2a9e..11997c85 100644 --- a/data/maldives_ecosystem_mapping/config_sentinel2.yaml +++ b/data/maldives_ecosystem_mapping/config_sentinel2.yaml @@ -30,7 +30,7 @@ model: data: class_path: rslearn.train.data_module.RslearnDataModule init_args: - path: gcs://rslearn-eai/datasets/maldives_ecosystem_mapping/dataset_v1/20240924/ + path: gcs://rslearn-eai/datasets/maldives_ecosystem_mapping/dataset_v1/20241017/ inputs: sentinel2: data_type: "raster" @@ -146,7 +146,7 @@ trainer: logging_interval: "epoch" - class_path: rslearn.train.prediction_writer.RslearnWriter init_args: - path: gcs://rslearn-eai/datasets/maldives_ecosystem_mapping/dataset_v1/20240924/ + path: gcs://rslearn-eai/datasets/maldives_ecosystem_mapping/dataset_v1/20241017/ output_layer: output selector: ["segment"] - class_path: lightning.pytorch.callbacks.ModelCheckpoint diff --git a/one_off_projects/minderoo/classify/config.yaml b/one_off_projects/minderoo/classify/config.yaml index 0b73483a..723a109b 100644 --- a/one_off_projects/minderoo/classify/config.yaml +++ b/one_off_projects/minderoo/classify/config.yaml @@ -17,14 +17,15 @@ model: num_fc_layers: 1 - class_path: rslearn.train.tasks.classification.ClassificationHead lr: 0.0001 - plateau_factor: 0.1 - plateau_patience: 10 + plateau: true + plateau_factor: 0.5 + plateau_patience: 2 plateau_min_lr: 0 - plateau_cooldown: 0 + plateau_cooldown: 5 data: class_path: rslearn.train.data_module.RslearnDataModule init_args: - root_dir: /data/favyenb/dvim/rslearn_classify/ + path: /data/favyenb/dvim/rslearn_classify/ inputs: image: data_type: "raster" @@ -88,8 +89,14 @@ trainer: logging_interval: "epoch" - class_path: rslearn.train.prediction_writer.RslearnWriter init_args: - root_dir: /data/favyenb/dvim/rslearn_classify/ + path: /data/favyenb/dvim/rslearn_classify/ output_layer: output selector: ["class"] + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + save_top_k: 1 + save_last: true + monitor: val_class/accuracy + mode: max rslp_project: minderoo_classify -rslp_experiment: data_20240811_satlaspretrain_flip_256_00 +rslp_experiment: data_20241028_satlaspretrain_flip_256_00 diff --git a/one_off_projects/minderoo/classify/config_infer.yaml b/one_off_projects/minderoo/classify/config_infer.yaml deleted file mode 100644 index c4972f8f..00000000 --- a/one_off_projects/minderoo/classify/config_infer.yaml +++ /dev/null @@ -1,95 +0,0 @@ -model: - class_path: rslearn.train.lightning_module.RslearnLightningModule - init_args: - model: - class_path: rslearn.models.multitask.MultiTaskModel - init_args: - encoder: - - class_path: rslearn.models.satlaspretrain.SatlasPretrain - init_args: - model_identifier: "Aerial_SwinB_SI" - decoders: - class: - - class_path: rslearn.models.pooling_decoder.PoolingDecoder - init_args: - in_channels: 1024 - out_channels: 2 - num_fc_layers: 1 - - class_path: rslearn.train.tasks.classification.ClassificationHead - lr: 0.0001 - plateau_factor: 0.1 - plateau_patience: 10 - plateau_min_lr: 0 - plateau_cooldown: 0 -data: - class_path: rslearn.train.data_module.RslearnDataModule - init_args: - root_dir: /data/favyenb/dvim/outputs.new/2024-08-11/2024-07-30_0.5/rslearn_dataset/ - inputs: - image: - data_type: "raster" - layers: ["maxar"] - bands: ["R", "G", "B"] - passthrough: true - label: - data_type: "vector" - layers: ["label"] - is_target: true - task: - class_path: rslearn.train.tasks.multi_task.MultiTask - init_args: - tasks: - class: - class_path: rslearn.train.tasks.classification.ClassificationTask - init_args: - property_name: "label" - classes: ["positive", "negative"] - prob_property: "prob" - input_mapping: - class: - label: "targets" - batch_size: 8 - num_workers: 16 - default_config: - transforms: - - class_path: rslearn.train.transforms.normalize.Normalize - init_args: - mean: 0 - std: 255 - train_config: - transforms: - - class_path: rslearn.train.transforms.normalize.Normalize - init_args: - mean: 0 - std: 255 - - class_path: rslearn.train.transforms.flip.Flip - tags: - split: train - sampler: - class_path: rslearn.train.dataset.WeightedRandomSamplerFactory - init_args: - option_key: "weight" - num_samples: 8192 - val_config: - tags: - split: val - test_config: - tags: - split: val - predict_config: - tags: - split: val - skip_targets: true -trainer: - max_epochs: 100 - callbacks: - - class_path: lightning.pytorch.callbacks.LearningRateMonitor - init_args: - logging_interval: "epoch" - - class_path: rslearn.train.prediction_writer.RslearnWriter - init_args: - root_dir: /data/favyenb/dvim/outputs.new/2024-08-11/2024-07-30_0.5/rslearn_dataset/ - output_layer: output - selector: ["class"] -rslp_project: minderoo_classify -rslp_experiment: data_20240811_satlaspretrain_flip_256_00 diff --git a/rslp/landsat_vessels/predict_pipeline.py b/rslp/landsat_vessels/predict_pipeline.py index d4668807..2e770051 100644 --- a/rslp/landsat_vessels/predict_pipeline.py +++ b/rslp/landsat_vessels/predict_pipeline.py @@ -215,7 +215,7 @@ def run_classifier( window.save_layer_datas(dict(LANDSAT_LAYER_NAME=layer_data)) logger.info("materialize dataset") - apply_windows_args = ApplyWindowsArgs(group=group) + apply_windows_args = ApplyWindowsArgs(group=group, workers=32) materialize_pipeline_args = MaterializePipelineArgs( disabled_layers=[], prepare_args=PrepareArgs(apply_windows_args=apply_windows_args), diff --git a/rslp/sentinel2_vessels/job_launcher.py b/rslp/sentinel2_vessels/job_launcher.py index c3630afa..81f517ae 100644 --- a/rslp/sentinel2_vessels/job_launcher.py +++ b/rslp/sentinel2_vessels/job_launcher.py @@ -61,10 +61,7 @@ def launch_job(image_name: str, tasks: list[PredictionTask]) -> None: ], constraints=Constraints( cluster=[ - # Don't use Renton cirrascale since it loads very slowly from Weka. - "ai2/prior-elanding", "ai2/jupiter-cirrascale-2", - "ai2/neptune-cirrascale", ] ), preemptible=True, diff --git a/rslp/sentinel2_vessels/predict_pipeline.py b/rslp/sentinel2_vessels/predict_pipeline.py index 81486c90..f1551a33 100644 --- a/rslp/sentinel2_vessels/predict_pipeline.py +++ b/rslp/sentinel2_vessels/predict_pipeline.py @@ -137,7 +137,7 @@ def get_vessel_detections( window.save_layer_datas(dict(SENTINEL2_LAYER_NAME=layer_data)) logger.info("Materialize dataset for Sentinel-2 Vessel Detection") - apply_windows_args = ApplyWindowsArgs(group=group, workers=1) + apply_windows_args = ApplyWindowsArgs(group=group, workers=32) materialize_pipeline_args = MaterializePipelineArgs( disabled_layers=[], prepare_args=PrepareArgs(apply_windows_args=apply_windows_args), @@ -297,9 +297,6 @@ def predict_pipeline(tasks: list[PredictionTask], scratch_path: str) -> None: with crop_fname.open("wb") as f: Image.fromarray(image.transpose(1, 2, 0)).save(f, format="PNG") - if scene_id not in json_vessels_by_scene: - json_vessels_by_scene[scene_id] = [] - json_vessels_by_scene[scene_id].append( dict( longitude=lon,