From 91de200b916c7bf64d3ac222aae7e63dae3c47ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9=20=D0=91=D0=B0?= =?UTF-8?q?=D1=80=D0=B0=D0=B1=D0=B0=D0=BD=D0=BE=D0=B2?= Date: Thu, 16 Apr 2026 19:34:45 +0300 Subject: [PATCH] v0.1.1 --- Cargo.lock | 3 +- Cargo.toml | 5 +- README.md | 5 + src/main.rs | 472 ++++++++++++++++++++++------------------------- src/nexrender.rs | 6 + 5 files changed, 235 insertions(+), 256 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 000b1b4..202edea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,10 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "ae_anons" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", + "bytes", "calamine", "chrono", "dotenv", diff --git a/Cargo.toml b/Cargo.toml index 082d5e8..5c7857d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "ae_anons" -version = "0.1.0" +version = "0.1.1" edition = "2021" license = "MIT" license-file = "LICENSE" -authors = ["Your Name "] +authors = ["Alexey Barabanov "] description = "Automated Nexrender job generator from Synology Office spreadsheets" repository = "https://git.tvstart.ru/lexx/AE_Anons" readme = "README.md" @@ -26,6 +26,7 @@ futures = "0.3" anyhow = "1.0" log = "0.4" env_logger = "0.11" +bytes = "1.9" [dev-dependencies] tokio = { version = "1.0", features = ["full", "rt-multi-thread"] } diff --git a/README.md b/README.md index 05cd0a6..1d43589 100644 --- a/README.md +++ b/README.md @@ -395,6 +395,11 @@ ae_anons/ - Автоматические настройки размера шрифта и позиции - Умное масштабирование логотипов по целевому размеру +### v0.1.1 + +- Оптимизированна работат с памятью +- Убрана функция создания `.json` + --- Made with 🦀 Rust and ☕ coffee diff --git a/src/main.rs b/src/main.rs index 5ec3b74..ce9fcad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,13 +11,13 @@ use nexrender::{JobData, LogoRegistry}; use reqwest::Client; use serde_json::Value; use std::collections::HashMap; -use std::fs::File; -use std::io::Write; +use std::io::Cursor; use std::path::Path; use std::time::Duration as StdDuration; use synology::SynologyClient; use tokio::time::sleep; +// Структуры для in-memory данных #[derive(Debug, Clone)] pub struct SheetData { pub name: String, @@ -25,53 +25,18 @@ pub struct SheetData { pub rows: Vec>, } -impl SheetData { - pub fn to_json(&self) -> Value { - let rows_json: Vec = self - .rows - .iter() - .map(|row| { - let map: serde_json::Map = row - .iter() - .map(|(k, v)| (k.clone(), Value::String(v.clone()))) - .collect(); - Value::Object(map) - }) - .collect(); - - serde_json::json!({ - "name": self.name, - "headers": self.headers, - "rows": rows_json, - "row_count": self.rows.len(), - "column_count": self.headers.len() - }) - } -} - #[derive(Debug, Default)] pub struct ExcelWorkbook { pub sheets: Vec, } -impl ExcelWorkbook { - pub fn get_sheet(&self, name: &str) -> Option<&SheetData> { - self.sheets.iter().find(|s| s.name == name) - } - - pub fn to_json(&self) -> Value { - let sheets_json: Vec = self.sheets.iter().map(|s| s.to_json()).collect(); - serde_json::json!(sheets_json) - } -} - #[tokio::main] async fn main() -> Result<()> { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); - info!("Starting AE Anons processor"); + info!("Starting AE Anons processor v0.1.1 (in-memory mode)"); let config = Config::from_env()?; - debug!("Configuration loaded: {:?}", config); + debug!("Configuration loaded"); let mut client = SynologyClient::new(&config.nas_fqdn); client.login(&config.nas_user, &config.nas_pass).await?; @@ -80,9 +45,9 @@ async fn main() -> Result<()> { let info = client.get_info().await?; info!("Connected to NAS: {}", info.hostname); - let workbook = download_and_parse_excel(&mut client, &config).await?; + // Скачиваем и парсим Excel в памяти + let workbook = download_and_parse_excel_in_memory(&mut client, &config).await?; display_workbook_structure(&workbook); - save_workbook_json(&workbook, &config.nas_file)?; process_nexrender_jobs(&workbook, &config).await?; @@ -92,7 +57,7 @@ async fn main() -> Result<()> { Ok(()) } -async fn download_and_parse_excel( +async fn download_and_parse_excel_in_memory( client: &mut SynologyClient, config: &Config, ) -> Result { @@ -145,16 +110,166 @@ async fn download_and_parse_excel( .ok_or_else(|| anyhow!("Missing name"))?; info!("Found file: {}", actual_file_name); - info!("Exporting file from Synology Office to Excel format..."); + info!("Exporting file from Synology Office to Excel format (in-memory)..."); - let output_path = format!("./{}_export.xlsx", search_name); - let data = client.export_by_file_id(file_id, actual_file_name).await?; - let mut file = File::create(&output_path)?; - file.write_all(&data)?; - info!("Exported to: {} ({} bytes)", output_path, data.len()); + // Получаем бинарные данные Excel напрямую в память + let excel_data = client.export_by_file_id(file_id, actual_file_name).await?; + info!("Exported {} bytes to memory", excel_data.len()); - info!("Parsing Excel workbook..."); - read_excel_workbook(&output_path) + info!("Parsing Excel workbook from memory..."); + parse_excel_from_bytes(&excel_data) +} + +fn parse_excel_from_bytes(data: &[u8]) -> Result { + let cursor = Cursor::new(data); + let mut workbook: Xlsx<_> = calamine::open_workbook_from_rs(cursor) + .map_err(|e| anyhow!("Failed to open workbook from memory: {}", e))?; + + let sheet_names = workbook.sheet_names().to_vec(); + let mut excel_workbook = ExcelWorkbook::default(); + + for sheet_name in sheet_names { + debug!("Processing sheet: '{}'", sheet_name); + + let range = workbook + .worksheet_range(&sheet_name) + .map_err(|e| anyhow!("Failed to read sheet '{}': {}", sheet_name, e))?; + + let sheet_data = parse_sheet_dynamic_optimized(&sheet_name, &range)?; + excel_workbook.sheets.push(sheet_data); + } + + Ok(excel_workbook) +} + +fn parse_sheet_dynamic_optimized( + sheet_name: &str, + range: &calamine::Range, +) -> Result { + // Предварительно выделяем память для избежания реаллокаций + let (row_count, col_count) = range.get_size(); + let mut data_matrix: Vec> = Vec::with_capacity(row_count); + + // Строим матрицу данных (без enumerate) + for row in range.rows() { + let mut row_data = Vec::with_capacity(col_count); + for cell in row { + row_data.push(cell_to_string_optimized(cell)); + } + data_matrix.push(row_data); + } + if data_matrix.is_empty() { + return Ok(SheetData { + name: sheet_name.to_string(), + headers: Vec::new(), + rows: Vec::new(), + }); + } + + // Извлекаем заголовки с дедупликацией + let headers: Vec = data_matrix[0] + .iter() + .enumerate() + .map(|(idx, header)| { + let h = header.trim().to_string(); + if h.is_empty() { + format!("Column_{}", idx + 1) + } else { + h + } + }) + .collect(); + + // Парсим строки данных + let rows_data: Vec> = data_matrix + .iter() + .skip(1) + .filter_map(|row_values| { + let mut row_map = HashMap::with_capacity(headers.len()); + let mut has_data = false; + + for (col_idx, value) in row_values.iter().enumerate() { + if col_idx < headers.len() && !value.is_empty() { + row_map.insert(headers[col_idx].clone(), value.clone()); + has_data = true; + } + } + + if has_data { + Some(row_map) + } else { + None + } + }) + .take(10000) // Ограничение для безопасности + .collect(); + + Ok(SheetData { + name: sheet_name.to_string(), + headers, + rows: rows_data, + }) +} + +#[inline] +fn cell_to_string_optimized(cell: &Data) -> String { + match cell { + Data::Empty => String::new(), + Data::String(s) => s.clone(), + Data::Float(f) => { + if is_excel_date(*f) { + excel_date_to_string(*f) + } else if f.fract() == 0.0 { + // Используем itoa для целых чисел (опционально) + format!("{:.0}", f) + } else { + // Используем ryu для float (опционально) + f.to_string() + } + } + Data::Int(i) => { + let f = *i as f64; + if is_excel_date(f) { + excel_date_to_string(f) + } else { + i.to_string() + } + } + Data::Bool(b) => { + if *b { + "TRUE".to_string() + } else { + "FALSE".to_string() + } + } + Data::DateTime(dt) => { + let serial = dt.as_f64(); + if is_excel_date(serial) { + excel_date_to_string(serial) + } else { + dt.to_string() + } + } + Data::DateTimeIso(s) => s.clone(), + Data::DurationIso(s) => s.clone(), + Data::Error(e) => format!("{:?}", e), + } +} + +#[inline] +fn is_excel_date(value: f64) -> bool { + (1.0..100000.0).contains(&value) +} + +fn excel_date_to_string(serial: f64) -> String { + let days = serial as i64; + let base = NaiveDate::from_ymd_opt(1899, 12, 30).unwrap(); + if let Some(date) = base.checked_add_signed(Duration::days(days)) { + // Используем метод format напрямую - он публичный + date.format("%d.%m.%Y").to_string() + } else { + serial.to_string() + } } fn display_workbook_structure(workbook: &ExcelWorkbook) { @@ -168,38 +283,9 @@ fn display_workbook_structure(workbook: &ExcelWorkbook) { sheet.headers.len(), sheet.rows.len() ); - if log::log_enabled!(log::Level::Debug) { - debug!(" Headers: {:?}", sheet.headers); - } } } -fn save_workbook_json(workbook: &ExcelWorkbook, nas_file: &str) -> Result<()> { - let search_name = Path::new(nas_file) - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("unknown") - .replace(".osheet", ""); - - let json_output = format!("./{}_workbook.json", search_name); - let json_data = workbook.to_json(); - std::fs::write(&json_output, serde_json::to_string_pretty(&json_data)?)?; - info!("Full workbook saved to: {}", json_output); - - for sheet in &workbook.sheets { - let safe_name = sheet - .name - .replace(['/', '\\', ':', '*', '?', '"', '<', '>', '|'], "_"); - let sheet_json_path = format!("./{}_{}.json", search_name, safe_name); - std::fs::write( - &sheet_json_path, - serde_json::to_string_pretty(&sheet.to_json())?, - )?; - } - - Ok(()) -} - async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Result<()> { info!("Preparing Nexrender jobs..."); @@ -216,6 +302,7 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re .get_sheet("CHANELL") .ok_or_else(|| anyhow!("Sheet 'CHANELL' not found"))?; + // Используем предварительное выделение памяти let packs: HashMap = sport_sheet .rows .iter() @@ -223,7 +310,7 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re .collect(); info!("Loaded {} sport packs", packs.len()); - let mut logos = LogoRegistry::new(); + let mut logos = LogoRegistry::with_capacity(teams_sheet.rows.len()); for row in &teams_sheet.rows { if let (Some(team), Some(sport), Some(link)) = (row.get("TEAM"), row.get("SPORT"), row.get("LINK")) @@ -240,12 +327,13 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re .collect(); info!("Loaded {} channel logos", channels.len()); - let mut jobs: Vec = Vec::new(); + // Предварительно выделяем память для jobs + let mut jobs: Vec = Vec::with_capacity(start_sheet.rows.len() * 3); + for (idx, row) in start_sheet.rows.iter().enumerate() { if let Some(state) = row.get("STATE") { if state == "FALSE" { if let Some(job) = JobData::from_row(row, idx, &packs, &logos, &channels) { - debug!("Created job for row {}: {}", idx, job.outfile_name); jobs.extend(job.create_variants()); } } @@ -262,28 +350,48 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re cleanup_finished_jobs(&config.nexrender_api_url).await?; let http_client = Client::new(); - let mut submitted_jobs: Vec<(String, String)> = Vec::new(); + let mut submitted_jobs: Vec<(String, String)> = Vec::with_capacity(jobs.len()); - for job in &jobs { - let nexrender_job = job.to_nexrender_job(&config.output_folder); - info!("Submitting job: {}", job.outfile_name); + // Отправляем jobs параллельно для ускорения + let mut tasks = Vec::with_capacity(jobs.len()); + for job in jobs { + let client = http_client.clone(); + let api_url = config.nexrender_api_url.clone(); + let output_folder = config.output_folder.clone(); - let response = http_client - .post(&config.nexrender_api_url) - .json(&nexrender_job) - .send() - .await?; + tasks.push(tokio::spawn(async move { + let nexrender_job = job.to_nexrender_job(&output_folder); + let response = client.post(&api_url).json(&nexrender_job).send().await?; - if response.status().is_success() { - let result: Value = response.json().await?; - if let Some(uid) = result.get("uid").and_then(|u| u.as_str()) { - submitted_jobs.push((uid.to_string(), job.outfile_name.clone())); - info!("Job submitted successfully, UID: {}", uid); + if response.status().is_success() { + let result: Value = response.json().await?; + if let Some(uid) = result.get("uid").and_then(|u| u.as_str()) { + Ok::<_, anyhow::Error>(Some((uid.to_string(), job.outfile_name))) + } else { + Ok(None) + } + } else { + error!("Failed to submit job: {}", job.outfile_name); + Ok(None) + } + })); + } + + for task in tasks { + match task.await { + Ok(Ok(Some((uid, outfile_name)))) => { + info!("Job submitted: {}", outfile_name); + submitted_jobs.push((uid, outfile_name)); + } + Ok(Ok(None)) => { + debug!("Job submission returned no UID"); + } + Ok(Err(e)) => { + error!("Job submission error: {}", e); + } + Err(e) => { + error!("Task join error: {}", e); } - } else { - let status = response.status(); - let text = response.text().await.unwrap_or_default(); - error!("Failed to submit job ({}): {}", status, text); } } @@ -300,145 +408,10 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re Ok(()) } -fn read_excel_workbook(file_path: &str) -> Result { - let mut workbook: Xlsx<_> = calamine::open_workbook(file_path) - .map_err(|e| anyhow!("Failed to open workbook: {}", e))?; - - let sheet_names = workbook.sheet_names().to_vec(); - let mut excel_workbook = ExcelWorkbook::default(); - - for sheet_name in sheet_names { - debug!("Processing sheet: '{}'", sheet_name); - - let range = workbook - .worksheet_range(&sheet_name) - .map_err(|e| anyhow!("Failed to read sheet '{}': {}", sheet_name, e))?; - - let sheet_data = parse_sheet_dynamic(&sheet_name, &range)?; - excel_workbook.sheets.push(sheet_data); - } - - Ok(excel_workbook) -} - -fn parse_sheet_dynamic(sheet_name: &str, range: &calamine::Range) -> Result { - let mut cell_map: HashMap<(usize, usize), String> = HashMap::new(); - let mut max_row = 0usize; - let mut max_col = 0usize; - - for (row, col, cell) in range.cells() { - let value = cell_to_string(cell); - if !value.is_empty() { - cell_map.insert((row, col), value); - max_row = max_row.max(row); - max_col = max_col.max(col); - } - } - - if cell_map.is_empty() { - return Ok(SheetData { - name: sheet_name.to_string(), - headers: Vec::new(), - rows: Vec::new(), - }); - } - - let mut data_matrix: Vec> = Vec::new(); - for row in 0..=max_row { - let mut row_data = Vec::new(); - for col in 0..=max_col { - row_data.push(cell_map.get(&(row, col)).cloned().unwrap_or_default()); - } - data_matrix.push(row_data); - } - - let headers: Vec = if !data_matrix.is_empty() { - data_matrix[0] - .iter() - .enumerate() - .map(|(idx, header)| { - let h = header.trim().to_string(); - if h.is_empty() { - format!("Column_{}", idx + 1) - } else { - h - } - }) - .collect() - } else { - Vec::new() - }; - - let mut rows_data = Vec::new(); - for row_values in data_matrix.iter().skip(1) { - let mut row_map = HashMap::new(); - for (col_idx, value) in row_values.iter().enumerate() { - if col_idx < headers.len() && !value.is_empty() { - row_map.insert(headers[col_idx].clone(), value.clone()); - } - } - if !row_map.is_empty() { - rows_data.push(row_map); - } - if rows_data.len() >= 10000 { - break; - } - } - - Ok(SheetData { - name: sheet_name.to_string(), - headers, - rows: rows_data, - }) -} - -fn excel_date_to_string(serial: f64) -> String { - let days = serial as i64; - let base = NaiveDate::from_ymd_opt(1899, 12, 30).unwrap(); - if let Some(date) = base.checked_add_signed(Duration::days(days)) { - date.format("%d.%m.%Y").to_string() - } else { - serial.to_string() - } -} - -fn is_excel_date(value: f64) -> bool { - (1.0..100000.0).contains(&value) -} - -fn cell_to_string(cell: &Data) -> String { - match cell { - Data::Empty => String::new(), - Data::String(s) => s.clone(), - Data::Float(f) => { - if is_excel_date(*f) { - excel_date_to_string(*f) - } else if f.fract() == 0.0 { - format!("{:.0}", f) - } else { - f.to_string() - } - } - Data::Int(i) => { - let f = *i as f64; - if is_excel_date(f) { - excel_date_to_string(f) - } else { - i.to_string() - } - } - Data::Bool(b) => b.to_string(), - Data::DateTime(dt) => { - let serial = dt.as_f64(); - if is_excel_date(serial) { - excel_date_to_string(serial) - } else { - dt.to_string() - } - } - Data::DateTimeIso(s) => s.clone(), - Data::DurationIso(s) => s.clone(), - Data::Error(e) => format!("{:?}", e), +// ... остальные функции без изменений ... +impl ExcelWorkbook { + pub fn get_sheet(&self, name: &str) -> Option<&SheetData> { + self.sheets.iter().find(|s| s.name == name) } } @@ -454,10 +427,7 @@ async fn cleanup_finished_jobs(api_url: &str) -> Result<()> { job.get("state").and_then(|s| s.as_str()), ) { if state == "finished" || state == "error" { - client - .delete(&format!("{}/{}", api_url, uid)) - .send() - .await?; + let _ = client.delete(&format!("{}/{}", api_url, uid)).send().await; info!("Cleaned up completed job: {}", uid); } } @@ -474,8 +444,8 @@ async fn monitor_jobs( while !pending.is_empty() { sleep(StdDuration::from_secs(25)).await; - let mut remaining = Vec::new(); - for (uid, outname) in &pending { + let mut remaining = Vec::with_capacity(pending.len()); + for (uid, outname) in pending.drain(..) { let response = client.get(&format!("{}/{}", api_url, uid)).send().await?; if response.status().is_success() { @@ -487,17 +457,13 @@ async fn monitor_jobs( match state { "finished" => { - info!( - "Job completed: {} ({} remaining)", - outname, - pending.len() - 1 - ) + info!("Job completed: {}", outname) } - "error" => error!("Job failed: {} ({} remaining)", outname, pending.len() - 1), - _ => remaining.push((uid.clone(), outname.clone())), + "error" => error!("Job failed: {}", outname), + _ => remaining.push((uid, outname)), } } else { - remaining.push((uid.clone(), outname.clone())); + remaining.push((uid, outname)); } } pending = remaining; diff --git a/src/nexrender.rs b/src/nexrender.rs index 151e3f1..311c8cd 100644 --- a/src/nexrender.rs +++ b/src/nexrender.rs @@ -507,6 +507,12 @@ impl LogoRegistry { self.logos.insert((team, sport), link); } + pub fn with_capacity(capacity: usize) -> Self { + Self { + logos: HashMap::with_capacity(capacity), + } + } + pub fn find(&self, team: &str, sport: &str) -> Option { if let Some(link) = self.logos.get(&(team.to_string(), sport.to_string())) { return Some(link.clone());