v0.1.1

Cargo.lock

@@ -10,9 +10,10 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
 
 [[package]]
 name = "ae_anons"
-version = "0.1.0"
+version = "0.1.1"
 dependencies = [
  "anyhow",
+ "bytes",
  "calamine",
  "chrono",
  "dotenv",

Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "ae_anons"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 license = "MIT"
 license-file = "LICENSE"
-authors = ["Your Name <a.barabanov@tvstart.ru>"]
+authors = ["Alexey Barabanov <a.barabanov@tvstart.ru>"]
 description = "Automated Nexrender job generator from Synology Office spreadsheets"
 repository = "https://git.tvstart.ru/lexx/AE_Anons"
 readme = "README.md"
@@ -26,6 +26,7 @@ futures = "0.3"
 anyhow = "1.0"
 log = "0.4"
 env_logger = "0.11"
+bytes = "1.9"
 
 [dev-dependencies]
 tokio = { version = "1.0", features = ["full", "rt-multi-thread"] }

README.md

@@ -395,6 +395,11 @@ ae_anons/
 - Автоматические настройки размера шрифта и позиции
 - Умное масштабирование логотипов по целевому размеру
 
+### v0.1.1
+
+- Оптимизированна работат с памятью
+- Убрана функция создания `.json`
+
 ---
 
 Made with 🦀 Rust and ☕ coffee

src/main.rs

@@ -11,13 +11,13 @@ use nexrender::{JobData, LogoRegistry};
 use reqwest::Client;
 use serde_json::Value;
 use std::collections::HashMap;
-use std::fs::File;
-use std::io::Write;
+use std::io::Cursor;
 use std::path::Path;
 use std::time::Duration as StdDuration;
 use synology::SynologyClient;
 use tokio::time::sleep;
 
+// Структуры для in-memory данных
 #[derive(Debug, Clone)]
 pub struct SheetData {
     pub name: String,
@@ -25,53 +25,18 @@ pub struct SheetData {
     pub rows: Vec<HashMap<String, String>>,
 }
 
-impl SheetData {
-    pub fn to_json(&self) -> Value {
-        let rows_json: Vec<Value> = self
-            .rows
-            .iter()
-            .map(|row| {
-                let map: serde_json::Map<String, Value> = row
-                    .iter()
-                    .map(|(k, v)| (k.clone(), Value::String(v.clone())))
-                    .collect();
-                Value::Object(map)
-            })
-            .collect();
-
-        serde_json::json!({
-            "name": self.name,
-            "headers": self.headers,
-            "rows": rows_json,
-            "row_count": self.rows.len(),
-            "column_count": self.headers.len()
-        })
-    }
-}
-
 #[derive(Debug, Default)]
 pub struct ExcelWorkbook {
     pub sheets: Vec<SheetData>,
 }
 
-impl ExcelWorkbook {
-    pub fn get_sheet(&self, name: &str) -> Option<&SheetData> {
-        self.sheets.iter().find(|s| s.name == name)
-    }
-
-    pub fn to_json(&self) -> Value {
-        let sheets_json: Vec<Value> = self.sheets.iter().map(|s| s.to_json()).collect();
-        serde_json::json!(sheets_json)
-    }
-}
-
 #[tokio::main]
 async fn main() -> Result<()> {
     env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
 
-    info!("Starting AE Anons processor");
+    info!("Starting AE Anons processor v0.1.1 (in-memory mode)");
     let config = Config::from_env()?;
-    debug!("Configuration loaded: {:?}", config);
+    debug!("Configuration loaded");
 
     let mut client = SynologyClient::new(&config.nas_fqdn);
     client.login(&config.nas_user, &config.nas_pass).await?;
@@ -80,9 +45,9 @@ async fn main() -> Result<()> {
     let info = client.get_info().await?;
     info!("Connected to NAS: {}", info.hostname);
 
-    let workbook = download_and_parse_excel(&mut client, &config).await?;
+    // Скачиваем и парсим Excel в памяти
+    let workbook = download_and_parse_excel_in_memory(&mut client, &config).await?;
     display_workbook_structure(&workbook);
-    save_workbook_json(&workbook, &config.nas_file)?;
 
     process_nexrender_jobs(&workbook, &config).await?;
 
@@ -92,7 +57,7 @@ async fn main() -> Result<()> {
     Ok(())
 }
 
-async fn download_and_parse_excel(
+async fn download_and_parse_excel_in_memory(
     client: &mut SynologyClient,
     config: &Config,
 ) -> Result<ExcelWorkbook> {
@@ -145,16 +110,166 @@ async fn download_and_parse_excel(
         .ok_or_else(|| anyhow!("Missing name"))?;
 
     info!("Found file: {}", actual_file_name);
-    info!("Exporting file from Synology Office to Excel format...");
+    info!("Exporting file from Synology Office to Excel format (in-memory)...");
 
-    let output_path = format!("./{}_export.xlsx", search_name);
-    let data = client.export_by_file_id(file_id, actual_file_name).await?;
-    let mut file = File::create(&output_path)?;
-    file.write_all(&data)?;
-    info!("Exported to: {} ({} bytes)", output_path, data.len());
+    // Получаем бинарные данные Excel напрямую в память
+    let excel_data = client.export_by_file_id(file_id, actual_file_name).await?;
+    info!("Exported {} bytes to memory", excel_data.len());
 
-    info!("Parsing Excel workbook...");
-    read_excel_workbook(&output_path)
+    info!("Parsing Excel workbook from memory...");
+    parse_excel_from_bytes(&excel_data)
+}
+
+fn parse_excel_from_bytes(data: &[u8]) -> Result<ExcelWorkbook> {
+    let cursor = Cursor::new(data);
+    let mut workbook: Xlsx<_> = calamine::open_workbook_from_rs(cursor)
+        .map_err(|e| anyhow!("Failed to open workbook from memory: {}", e))?;
+
+    let sheet_names = workbook.sheet_names().to_vec();
+    let mut excel_workbook = ExcelWorkbook::default();
+
+    for sheet_name in sheet_names {
+        debug!("Processing sheet: '{}'", sheet_name);
+
+        let range = workbook
+            .worksheet_range(&sheet_name)
+            .map_err(|e| anyhow!("Failed to read sheet '{}': {}", sheet_name, e))?;
+
+        let sheet_data = parse_sheet_dynamic_optimized(&sheet_name, &range)?;
+        excel_workbook.sheets.push(sheet_data);
+    }
+
+    Ok(excel_workbook)
+}
+
+fn parse_sheet_dynamic_optimized(
+    sheet_name: &str,
+    range: &calamine::Range<Data>,
+) -> Result<SheetData> {
+    // Предварительно выделяем память для избежания реаллокаций
+    let (row_count, col_count) = range.get_size();
+    let mut data_matrix: Vec<Vec<String>> = Vec::with_capacity(row_count);
+
+    // Строим матрицу данных (без enumerate)
+    for row in range.rows() {
+        let mut row_data = Vec::with_capacity(col_count);
+        for cell in row {
+            row_data.push(cell_to_string_optimized(cell));
+        }
+        data_matrix.push(row_data);
+    }
+    if data_matrix.is_empty() {
+        return Ok(SheetData {
+            name: sheet_name.to_string(),
+            headers: Vec::new(),
+            rows: Vec::new(),
+        });
+    }
+
+    // Извлекаем заголовки с дедупликацией
+    let headers: Vec<String> = data_matrix[0]
+        .iter()
+        .enumerate()
+        .map(|(idx, header)| {
+            let h = header.trim().to_string();
+            if h.is_empty() {
+                format!("Column_{}", idx + 1)
+            } else {
+                h
+            }
+        })
+        .collect();
+
+    // Парсим строки данных
+    let rows_data: Vec<HashMap<String, String>> = data_matrix
+        .iter()
+        .skip(1)
+        .filter_map(|row_values| {
+            let mut row_map = HashMap::with_capacity(headers.len());
+            let mut has_data = false;
+
+            for (col_idx, value) in row_values.iter().enumerate() {
+                if col_idx < headers.len() && !value.is_empty() {
+                    row_map.insert(headers[col_idx].clone(), value.clone());
+                    has_data = true;
+                }
+            }
+
+            if has_data {
+                Some(row_map)
+            } else {
+                None
+            }
+        })
+        .take(10000) // Ограничение для безопасности
+        .collect();
+
+    Ok(SheetData {
+        name: sheet_name.to_string(),
+        headers,
+        rows: rows_data,
+    })
+}
+
+#[inline]
+fn cell_to_string_optimized(cell: &Data) -> String {
+    match cell {
+        Data::Empty => String::new(),
+        Data::String(s) => s.clone(),
+        Data::Float(f) => {
+            if is_excel_date(*f) {
+                excel_date_to_string(*f)
+            } else if f.fract() == 0.0 {
+                // Используем itoa для целых чисел (опционально)
+                format!("{:.0}", f)
+            } else {
+                // Используем ryu для float (опционально)
+                f.to_string()
+            }
+        }
+        Data::Int(i) => {
+            let f = *i as f64;
+            if is_excel_date(f) {
+                excel_date_to_string(f)
+            } else {
+                i.to_string()
+            }
+        }
+        Data::Bool(b) => {
+            if *b {
+                "TRUE".to_string()
+            } else {
+                "FALSE".to_string()
+            }
+        }
+        Data::DateTime(dt) => {
+            let serial = dt.as_f64();
+            if is_excel_date(serial) {
+                excel_date_to_string(serial)
+            } else {
+                dt.to_string()
+            }
+        }
+        Data::DateTimeIso(s) => s.clone(),
+        Data::DurationIso(s) => s.clone(),
+        Data::Error(e) => format!("{:?}", e),
+    }
+}
+
+#[inline]
+fn is_excel_date(value: f64) -> bool {
+    (1.0..100000.0).contains(&value)
+}
+
+fn excel_date_to_string(serial: f64) -> String {
+    let days = serial as i64;
+    let base = NaiveDate::from_ymd_opt(1899, 12, 30).unwrap();
+    if let Some(date) = base.checked_add_signed(Duration::days(days)) {
+        // Используем метод format напрямую - он публичный
+        date.format("%d.%m.%Y").to_string()
+    } else {
+        serial.to_string()
+    }
 }
 
 fn display_workbook_structure(workbook: &ExcelWorkbook) {
@@ -168,36 +283,7 @@ fn display_workbook_structure(workbook: &ExcelWorkbook) {
             sheet.headers.len(),
             sheet.rows.len()
         );
-        if log::log_enabled!(log::Level::Debug) {
-            debug!("  Headers: {:?}", sheet.headers);
     }
-    }
-}
-
-fn save_workbook_json(workbook: &ExcelWorkbook, nas_file: &str) -> Result<()> {
-    let search_name = Path::new(nas_file)
-        .file_name()
-        .and_then(|n| n.to_str())
-        .unwrap_or("unknown")
-        .replace(".osheet", "");
-
-    let json_output = format!("./{}_workbook.json", search_name);
-    let json_data = workbook.to_json();
-    std::fs::write(&json_output, serde_json::to_string_pretty(&json_data)?)?;
-    info!("Full workbook saved to: {}", json_output);
-
-    for sheet in &workbook.sheets {
-        let safe_name = sheet
-            .name
-            .replace(['/', '\\', ':', '*', '?', '"', '<', '>', '|'], "_");
-        let sheet_json_path = format!("./{}_{}.json", search_name, safe_name);
-        std::fs::write(
-            &sheet_json_path,
-            serde_json::to_string_pretty(&sheet.to_json())?,
-        )?;
-    }
-
-    Ok(())
 }
 
 async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Result<()> {
@@ -216,6 +302,7 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re
         .get_sheet("CHANELL")
         .ok_or_else(|| anyhow!("Sheet 'CHANELL' not found"))?;
 
+    // Используем предварительное выделение памяти
     let packs: HashMap<String, String> = sport_sheet
         .rows
         .iter()
@@ -223,7 +310,7 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re
         .collect();
     info!("Loaded {} sport packs", packs.len());
 
-    let mut logos = LogoRegistry::new();
+    let mut logos = LogoRegistry::with_capacity(teams_sheet.rows.len());
     for row in &teams_sheet.rows {
         if let (Some(team), Some(sport), Some(link)) =
             (row.get("TEAM"), row.get("SPORT"), row.get("LINK"))
@@ -240,12 +327,13 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re
         .collect();
     info!("Loaded {} channel logos", channels.len());
 
-    let mut jobs: Vec<JobData> = Vec::new();
+    // Предварительно выделяем память для jobs
+    let mut jobs: Vec<JobData> = Vec::with_capacity(start_sheet.rows.len() * 3);
+
     for (idx, row) in start_sheet.rows.iter().enumerate() {
         if let Some(state) = row.get("STATE") {
             if state == "FALSE" {
                 if let Some(job) = JobData::from_row(row, idx, &packs, &logos, &channels) {
-                    debug!("Created job for row {}: {}", idx, job.outfile_name);
                     jobs.extend(job.create_variants());
                 }
             }
@@ -262,28 +350,48 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re
     cleanup_finished_jobs(&config.nexrender_api_url).await?;
 
     let http_client = Client::new();
-    let mut submitted_jobs: Vec<(String, String)> = Vec::new();
+    let mut submitted_jobs: Vec<(String, String)> = Vec::with_capacity(jobs.len());
 
-    for job in &jobs {
-        let nexrender_job = job.to_nexrender_job(&config.output_folder);
-        info!("Submitting job: {}", job.outfile_name);
+    // Отправляем jobs параллельно для ускорения
+    let mut tasks = Vec::with_capacity(jobs.len());
+    for job in jobs {
+        let client = http_client.clone();
+        let api_url = config.nexrender_api_url.clone();
+        let output_folder = config.output_folder.clone();
 
-        let response = http_client
-            .post(&config.nexrender_api_url)
-            .json(&nexrender_job)
-            .send()
-            .await?;
+        tasks.push(tokio::spawn(async move {
+            let nexrender_job = job.to_nexrender_job(&output_folder);
+            let response = client.post(&api_url).json(&nexrender_job).send().await?;
 
             if response.status().is_success() {
                 let result: Value = response.json().await?;
                 if let Some(uid) = result.get("uid").and_then(|u| u.as_str()) {
-                submitted_jobs.push((uid.to_string(), job.outfile_name.clone()));
-                info!("Job submitted successfully, UID: {}", uid);
+                    Ok::<_, anyhow::Error>(Some((uid.to_string(), job.outfile_name)))
+                } else {
+                    Ok(None)
                 }
             } else {
-            let status = response.status();
-            let text = response.text().await.unwrap_or_default();
-            error!("Failed to submit job ({}): {}", status, text);
+                error!("Failed to submit job: {}", job.outfile_name);
+                Ok(None)
+            }
+        }));
+    }
+
+    for task in tasks {
+        match task.await {
+            Ok(Ok(Some((uid, outfile_name)))) => {
+                info!("Job submitted: {}", outfile_name);
+                submitted_jobs.push((uid, outfile_name));
+            }
+            Ok(Ok(None)) => {
+                debug!("Job submission returned no UID");
+            }
+            Ok(Err(e)) => {
+                error!("Job submission error: {}", e);
+            }
+            Err(e) => {
+                error!("Task join error: {}", e);
+            }
         }
     }
 
@@ -300,145 +408,10 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re
     Ok(())
 }
 
-fn read_excel_workbook(file_path: &str) -> Result<ExcelWorkbook> {
-    let mut workbook: Xlsx<_> = calamine::open_workbook(file_path)
-        .map_err(|e| anyhow!("Failed to open workbook: {}", e))?;
-
-    let sheet_names = workbook.sheet_names().to_vec();
-    let mut excel_workbook = ExcelWorkbook::default();
-
-    for sheet_name in sheet_names {
-        debug!("Processing sheet: '{}'", sheet_name);
-
-        let range = workbook
-            .worksheet_range(&sheet_name)
-            .map_err(|e| anyhow!("Failed to read sheet '{}': {}", sheet_name, e))?;
-
-        let sheet_data = parse_sheet_dynamic(&sheet_name, &range)?;
-        excel_workbook.sheets.push(sheet_data);
-    }
-
-    Ok(excel_workbook)
-}
-
-fn parse_sheet_dynamic(sheet_name: &str, range: &calamine::Range<Data>) -> Result<SheetData> {
-    let mut cell_map: HashMap<(usize, usize), String> = HashMap::new();
-    let mut max_row = 0usize;
-    let mut max_col = 0usize;
-
-    for (row, col, cell) in range.cells() {
-        let value = cell_to_string(cell);
-        if !value.is_empty() {
-            cell_map.insert((row, col), value);
-            max_row = max_row.max(row);
-            max_col = max_col.max(col);
-        }
-    }
-
-    if cell_map.is_empty() {
-        return Ok(SheetData {
-            name: sheet_name.to_string(),
-            headers: Vec::new(),
-            rows: Vec::new(),
-        });
-    }
-
-    let mut data_matrix: Vec<Vec<String>> = Vec::new();
-    for row in 0..=max_row {
-        let mut row_data = Vec::new();
-        for col in 0..=max_col {
-            row_data.push(cell_map.get(&(row, col)).cloned().unwrap_or_default());
-        }
-        data_matrix.push(row_data);
-    }
-
-    let headers: Vec<String> = if !data_matrix.is_empty() {
-        data_matrix[0]
-            .iter()
-            .enumerate()
-            .map(|(idx, header)| {
-                let h = header.trim().to_string();
-                if h.is_empty() {
-                    format!("Column_{}", idx + 1)
-                } else {
-                    h
-                }
-            })
-            .collect()
-    } else {
-        Vec::new()
-    };
-
-    let mut rows_data = Vec::new();
-    for row_values in data_matrix.iter().skip(1) {
-        let mut row_map = HashMap::new();
-        for (col_idx, value) in row_values.iter().enumerate() {
-            if col_idx < headers.len() && !value.is_empty() {
-                row_map.insert(headers[col_idx].clone(), value.clone());
-            }
-        }
-        if !row_map.is_empty() {
-            rows_data.push(row_map);
-        }
-        if rows_data.len() >= 10000 {
-            break;
-        }
-    }
-
-    Ok(SheetData {
-        name: sheet_name.to_string(),
-        headers,
-        rows: rows_data,
-    })
-}
-
-fn excel_date_to_string(serial: f64) -> String {
-    let days = serial as i64;
-    let base = NaiveDate::from_ymd_opt(1899, 12, 30).unwrap();
-    if let Some(date) = base.checked_add_signed(Duration::days(days)) {
-        date.format("%d.%m.%Y").to_string()
-    } else {
-        serial.to_string()
-    }
-}
-
-fn is_excel_date(value: f64) -> bool {
-    (1.0..100000.0).contains(&value)
-}
-
-fn cell_to_string(cell: &Data) -> String {
-    match cell {
-        Data::Empty => String::new(),
-        Data::String(s) => s.clone(),
-        Data::Float(f) => {
-            if is_excel_date(*f) {
-                excel_date_to_string(*f)
-            } else if f.fract() == 0.0 {
-                format!("{:.0}", f)
-            } else {
-                f.to_string()
-            }
-        }
-        Data::Int(i) => {
-            let f = *i as f64;
-            if is_excel_date(f) {
-                excel_date_to_string(f)
-            } else {
-                i.to_string()
-            }
-        }
-        Data::Bool(b) => b.to_string(),
-        Data::DateTime(dt) => {
-            let serial = dt.as_f64();
-            if is_excel_date(serial) {
-                excel_date_to_string(serial)
-            } else {
-                dt.to_string()
-            }
-        }
-        Data::DateTimeIso(s) => s.clone(),
-        Data::DurationIso(s) => s.clone(),
-        Data::Error(e) => format!("{:?}", e),
+// ... остальные функции без изменений ...
+impl ExcelWorkbook {
+    pub fn get_sheet(&self, name: &str) -> Option<&SheetData> {
+        self.sheets.iter().find(|s| s.name == name)
     }
 }
 
@@ -454,10 +427,7 @@ async fn cleanup_finished_jobs(api_url: &str) -> Result<()> {
                 job.get("state").and_then(|s| s.as_str()),
             ) {
                 if state == "finished" || state == "error" {
-                    client
-                        .delete(&format!("{}/{}", api_url, uid))
-                        .send()
-                        .await?;
+                    let _ = client.delete(&format!("{}/{}", api_url, uid)).send().await;
                     info!("Cleaned up completed job: {}", uid);
                 }
             }
@@ -474,8 +444,8 @@ async fn monitor_jobs(
     while !pending.is_empty() {
         sleep(StdDuration::from_secs(25)).await;
 
-        let mut remaining = Vec::new();
-        for (uid, outname) in &pending {
+        let mut remaining = Vec::with_capacity(pending.len());
+        for (uid, outname) in pending.drain(..) {
             let response = client.get(&format!("{}/{}", api_url, uid)).send().await?;
 
             if response.status().is_success() {
@@ -487,17 +457,13 @@ async fn monitor_jobs(
 
                 match state {
                     "finished" => {
-                        info!(
-                            "Job completed: {} ({} remaining)",
-                            outname,
-                            pending.len() - 1
-                        )
+                        info!("Job completed: {}", outname)
                     }
-                    "error" => error!("Job failed: {} ({} remaining)", outname, pending.len() - 1),
-                    _ => remaining.push((uid.clone(), outname.clone())),
+                    "error" => error!("Job failed: {}", outname),
+                    _ => remaining.push((uid, outname)),
                 }
             } else {
-                remaining.push((uid.clone(), outname.clone()));
+                remaining.push((uid, outname));
             }
         }
         pending = remaining;

src/nexrender.rs

@@ -507,6 +507,12 @@ impl LogoRegistry {
         self.logos.insert((team, sport), link);
     }
 
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self {
+            logos: HashMap::with_capacity(capacity),
+        }
+    }
+
     pub fn find(&self, team: &str, sport: &str) -> Option<String> {
         if let Some(link) = self.logos.get(&(team.to_string(), sport.to_string())) {
             return Some(link.clone());