{ "cells": [ { "cell_type": "markdown", "id": "ca9fa3cb-0c55-4eb3-98f6-b9ef4374f137", "metadata": {}, "source": [ "This notebook contains an algorithm to add repetition counts for field files where it wasn't manually entered and gets rid of commas that might cause issues when saving to a CSV" ] }, { "cell_type": "code", "execution_count": 1, "id": "a6d20f54-3922-424b-9637-6705e20d9ac0", "metadata": {}, "outputs": [], "source": [ "import os\n", "import glob\n", "import pandas as pd\n", "import numpy as np\n", "from openpyxl import load_workbook\n", "import csv\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import datetime as dt" ] }, { "cell_type": "code", "execution_count": 2, "id": "bfea848d-2ec6-4b5c-909e-dbcded7837e0", "metadata": {}, "outputs": [], "source": [ "v0_path = r\"C:\\Users\\Nora Greenwood\\Desktop\\CONTRASTS\\processing_albedo_data\\v0_CONTRASTS_albedo\"\n", "v1_path = r\"C:\\Users\\Nora Greenwood\\Desktop\\CONTRASTS\\processing_albedo_data\\v1_CONTRASTS_albedo\\albedonotes\"\n", "\n", "notes_dirs = [subdir for subdir in os.listdir(v0_path)\n", " if os.path.isdir(os.path.join(v0_path, subdir)) and\n", " any(fname.startswith(\"alb\") for fname in os.listdir(os.path.join(v0_path, subdir)))]" ] }, { "cell_type": "code", "execution_count": 3, "id": "a9f0b423-c3f5-4827-a4cf-1e25a78941a6", "metadata": {}, "outputs": [], "source": [ "def add_repetition(site, out_path):\n", " notes_path = os.path.join(v0_path, site, 'albedonotes-contrasts-'+site+'.xlsx')\n", "\n", " # read in notes and pull in position data\n", " notes = pd.read_excel(notes_path,header=2)\n", " if 'Repetition' in notes.columns:\n", " print('manually done for '+site)\n", " else:\n", " positions = list(notes.Position)\n", " counts = {}\n", " repetition = []\n", " for p in positions:\n", " counts[p] = counts.get(p,0)+1\n", " repetition.append(counts[p])\n", " notes.insert(13, 'Repetition', repetition)\n", "\n", " # save to v1\n", " #read in met file again to get additional header information\n", " header = pd.read_excel(notes_path,header=None)\n", " header = header.fillna('NaN')\n", " header = header.iloc[0:2,0:6]\n", " # get rid of commas in the extra notes before saving to csv\n", " if not pd.isnull(header.iloc[1,5]):\n", " header.iloc[1,5] = header.iloc[1,5].replace(',',';')\n", " header.iloc[0,3] = header.iloc[0,3].replace(',',';')\n", "\n", " newname ='albedonotes-contrasts-'+site+'-v1.csv'\n", " newfilename = os.path.join(out_path, newname)\n", "\n", " header.to_csv(newfilename, header=False, index=False, mode='w')\n", " \n", " # write the rest\n", " notes.to_csv(newfilename, header=True,index=False, mode='a')\n", "\n", " print('wrote: '+newname)" ] }, { "cell_type": "code", "execution_count": 4, "id": "5978f1a7-8ef2-4dd8-8446-74d73e43cbb3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "wrote: albedonotes-contrasts-20250710-r1a-R1_mainATransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250711-r1a-ponds-library-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250711-r1a-R1_mainATransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250712-r1a-ponds-colocated-v1.csv\n", "\n", "manually done for 20250712-r1a-ssl-experiment-am1\n", "wrote: albedonotes-contrasts-20250712-r1a-ssl-experiment-am1-v1.csv\n", "\n", "manually done for 20250712-r1a-ssl-experiment-pm1\n", "wrote: albedonotes-contrasts-20250712-r1a-ssl-experiment-pm1-v1.csv\n", "\n", "manually done for 20250712-r1a-ssl-experiment-pm2\n", "wrote: albedonotes-contrasts-20250712-r1a-ssl-experiment-pm2-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250715-r2a-R2_mainTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250715-r2a-R2_opticsTransect-v1.csv\n", "\n", "manually done for 20250715-r2a-ssl1-experiment-am1\n", "wrote: albedonotes-contrasts-20250715-r2a-ssl1-experiment-am1-v1.csv\n", "\n", "manually done for 20250715-r2a-ssl1-experiment-pm1\n", "wrote: albedonotes-contrasts-20250715-r2a-ssl1-experiment-pm1-v1.csv\n", "\n", "manually done for 20250715-r2a-ssl1-experiment-pm2\n", "wrote: albedonotes-contrasts-20250715-r2a-ssl1-experiment-pm2-v1.csv\n", "\n", "manually done for 20250716-r2a-ssl1-experiment-am1\n", "wrote: albedonotes-contrasts-20250716-r2a-ssl1-experiment-am1-v1.csv\n", "\n", "manually done for 20250716-r2a-ssl1-experiment-pm1\n", "wrote: albedonotes-contrasts-20250716-r2a-ssl1-experiment-pm1-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250717-r2a-ponds-library-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250717-r2a-R2_raftedIceTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250717-r2a-snow-colocated-v1.csv\n", "\n", "manually done for 20250717-r2a-ssl1-experiment-pm1\n", "wrote: albedonotes-contrasts-20250717-r2a-ssl1-experiment-pm1-v1.csv\n", "\n", "manually done for 20250717-r2a-ssl1-experiment-pm2\n", "wrote: albedonotes-contrasts-20250717-r2a-ssl1-experiment-pm2-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250720-r3a-R3_mainTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250720-r3a-R3_opticsTransect-v1.csv\n", "\n", "manually done for 20250721-r3a-ssl-experiment-am1\n", "wrote: albedonotes-contrasts-20250721-r3a-ssl-experiment-am1-v1.csv\n", "\n", "manually done for 20250721-r3a-ssl-experiment-pm1\n", "wrote: albedonotes-contrasts-20250721-r3a-ssl-experiment-pm1-v1.csv\n", "\n", "manually done for 20250721-r3a-ssl-experiment-pm2\n", "wrote: albedonotes-contrasts-20250721-r3a-ssl-experiment-pm2-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250722-r3a-ponds-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250722-r3a-ponds-library-v1.csv\n", "\n", "manually done for 20250722-r3a-ssl-experiment-pm1\n", "wrote: albedonotes-contrasts-20250722-r3a-ssl-experiment-pm1-v1.csv\n", "\n", "manually done for 20250722-r3a-ssl-experiment-pm2\n", "wrote: albedonotes-contrasts-20250722-r3a-ssl-experiment-pm2-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250726-r1b-ponds-library-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250726-r1b-R1_mainBTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250727-r1b-metcity-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250727-r1b-ponds-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250727-r1b-ponds-library-am-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250727-r1b-ponds-library-pm-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250727-r1b-R1_opticsBTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250727-r1b-snow-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250728-r1b-ponds-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250730-r2b-R2_opticsTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250731-r2b-metcity-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250731-r2b-ponds-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250731-r2b-ponds-library-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250731-r2b-R2_raftedIceTransect-v1.csv\n", "\n", "manually done for 20250731-r2b-ssl2-experiment\n", "wrote: albedonotes-contrasts-20250731-r2b-ssl2-experiment-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250801-r2b-ponds-library-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250801-r2b-R2_mainTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250801-r2b-R2_opticsTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250801-r2b-R2_raftedIceTransect-v1.csv\n", "\n", "manually done for 20250801-r2b-ssl2-experiment\n", "wrote: albedonotes-contrasts-20250801-r2b-ssl2-experiment-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250804-r3b-ponds-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250804-r3b-R3_mainTransect-v1.csv\n", "\n", "manually done for 20250804-r3b-ssl-experiment\n", "wrote: albedonotes-contrasts-20250804-r3b-ssl-experiment-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250805-r3b-ponds-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250805-r3b-R3_opticsTransect-am-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250805-r3b-R3_opticsTransect-pm-v1.csv\n", "\n", "manually done for 20250805-r3b-ssl-experiment-am1\n", "wrote: albedonotes-contrasts-20250805-r3b-ssl-experiment-am1-v1.csv\n", "\n", "manually done for 20250805-r3b-ssl-experiment-pm1\n", "wrote: albedonotes-contrasts-20250805-r3b-ssl-experiment-pm1-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250806-r3b-metcity-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250806-r3b-ponds-colocated-v1.csv\n", "\n", "manually done for 20250806-r3b-ssl-experiment\n", "wrote: albedonotes-contrasts-20250806-r3b-ssl-experiment-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250809-r1c-R1_mainCTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250810-r1c-coring-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250810-r1c-ponds-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250810-r1c-R1_mainCTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250810-r1c-R1_opticsCTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250812-r2c-R2_raftedIceTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250812-r2c-R2_tTransect-v1.csv\n", "\n", "manually done for 20250812-r2c-ssl3-experiment\n", "wrote: albedonotes-contrasts-20250812-r2c-ssl3-experiment-v1.csv\n", "\n", "manually done for 20250813-r2c-coring-colocated\n", "wrote: albedonotes-contrasts-20250813-r2c-coring-colocated-v1.csv\n", "\n", "manually done for 20250813-r2c-ssl3-experiment-am1\n", "wrote: albedonotes-contrasts-20250813-r2c-ssl3-experiment-am1-v1.csv\n", "\n", "manually done for 20250813-r2c-ssl3-experiment-pm1\n", "wrote: albedonotes-contrasts-20250813-r2c-ssl3-experiment-pm1-v1.csv\n", "\n", "manually done for 20250814-r2c-coring-colocated\n", "wrote: albedonotes-contrasts-20250814-r2c-coring-colocated-v1.csv\n", "\n", "manually done for 20250814-r2c-snow-colocated\n", "wrote: albedonotes-contrasts-20250814-r2c-snow-colocated-v1.csv\n", "\n", "manually done for 20250814-r2c-ssl3-experiment-am1\n", "wrote: albedonotes-contrasts-20250814-r2c-ssl3-experiment-am1-v1.csv\n", "\n", "manually done for 20250814-r2c-ssl3-experiment-pm1\n", "wrote: albedonotes-contrasts-20250814-r2c-ssl3-experiment-pm1-v1.csv\n", "\n", "manually done for 20250817-r3c-ponds-colocated\n", "wrote: albedonotes-contrasts-20250817-r3c-ponds-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250817-r3c-R3_mainTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250817-r3c-R3_opticsTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250823-r2d-beartracks-library-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250823-r2d-coring-colocated-v1.csv\n", "\n", "manually done for 20250823-r2d-ponds-library\n", "wrote: albedonotes-contrasts-20250823-r2d-ponds-library-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250823-r2d-R2_raftedIceTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250823-r2d-R2_tTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250824-r2d-remotesensing-colocated-v1.csv\n", "\n", "manually done for 20250824-r2d-snow-colocated\n", "wrote: albedonotes-contrasts-20250824-r2d-snow-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250827-r3d-ponds-colocated-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250827-r3d-R3_mainTransect-v1.csv\n", "\n", "wrote: albedonotes-contrasts-20250827-r3d-R3_opticsTransect-v1.csv\n", "\n" ] } ], "source": [ "# iterate through notes\n", "for i in range(len(notes_dirs)):\n", " add_repetition(notes_dirs[i], v1_path)\n", " print('')" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.14.3" } }, "nbformat": 4, "nbformat_minor": 5 }