lexx/AE_Anons
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases 3 Activity

v0.1.1

Browse Source
This commit is contained in:
Алексей Барабанов
2026-04-16 19:34:45 +03:00
parent 1e1d9a9935
commit 91de200b91
5 changed files with 235 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@@ -10,9 +10,10 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "ae_anons"
version = "0.1.0"
version = "0.1.1"
dependencies = [
"anyhow",
"bytes",
"calamine",
"chrono",
"dotenv",

5
Cargo.toml
View File

@@ -1,10 +1,10 @@
[package]
name = "ae_anons"
version = "0.1.0"
version = "0.1.1"
edition = "2021"
license = "MIT"
license-file = "LICENSE"
authors = ["Your Name <a.barabanov@tvstart.ru>"]
authors = ["Alexey Barabanov <a.barabanov@tvstart.ru>"]
description = "Automated Nexrender job generator from Synology Office spreadsheets"
repository = "https://git.tvstart.ru/lexx/AE_Anons"
readme = "README.md"
@@ -26,6 +26,7 @@ futures = "0.3"
anyhow = "1.0"
log = "0.4"
env_logger = "0.11"
bytes = "1.9"
[dev-dependencies]
tokio = { version = "1.0", features = ["full", "rt-multi-thread"] }

5
README.md
View File

@@ -395,6 +395,11 @@ ae_anons/
- Автоматические настройки размера шрифта и позиции
- Умное масштабирование логотипов по целевому размеру
### v0.1.1
- Оптимизированна работат с памятью
- Убрана функция создания `.json`
---
Made with 🦀 Rust and ☕ coffee

472
src/main.rs
View File

@@ -11,13 +11,13 @@ use nexrender::{JobData, LogoRegistry};
use reqwest::Client;
use serde_json::Value;
use std::collections::HashMap;
use std::fs::File;
use std::io::Write;
use std::io::Cursor;
use std::path::Path;
use std::time::Duration as StdDuration;
use synology::SynologyClient;
use tokio::time::sleep;
// Структуры для in-memory данных
#[derive(Debug, Clone)]
pub struct SheetData {
pub name: String,
@@ -25,53 +25,18 @@ pub struct SheetData {
pub rows: Vec<HashMap<String, String>>,
}
impl SheetData {
pub fn to_json(&self) -> Value {
let rows_json: Vec<Value> = self
.rows
.iter()
.map(|row| {
let map: serde_json::Map<String, Value> = row
.iter()
.map(|(k, v)| (k.clone(), Value::String(v.clone())))
.collect();
Value::Object(map)
})
.collect();
serde_json::json!({
"name": self.name,
"headers": self.headers,
"rows": rows_json,
"row_count": self.rows.len(),
"column_count": self.headers.len()
})
}
}
#[derive(Debug, Default)]
pub struct ExcelWorkbook {
pub sheets: Vec<SheetData>,
}
impl ExcelWorkbook {
pub fn get_sheet(&self, name: &str) -> Option<&SheetData> {
self.sheets.iter().find(|s| s.name == name)
}
pub fn to_json(&self) -> Value {
let sheets_json: Vec<Value> = self.sheets.iter().map(|s| s.to_json()).collect();
serde_json::json!(sheets_json)
}
}
#[tokio::main]
async fn main() -> Result<()> {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
info!("Starting AE Anons processor");
info!("Starting AE Anons processor v0.1.1 (in-memory mode)");
let config = Config::from_env()?;
debug!("Configuration loaded: {:?}", config);
debug!("Configuration loaded");
let mut client = SynologyClient::new(&config.nas_fqdn);
client.login(&config.nas_user, &config.nas_pass).await?;
@@ -80,9 +45,9 @@ async fn main() -> Result<()> {
let info = client.get_info().await?;
info!("Connected to NAS: {}", info.hostname);
let workbook = download_and_parse_excel(&mut client, &config).await?;
// Скачиваем и парсим Excel в памяти
let workbook = download_and_parse_excel_in_memory(&mut client, &config).await?;
display_workbook_structure(&workbook);
save_workbook_json(&workbook, &config.nas_file)?;
process_nexrender_jobs(&workbook, &config).await?;
@@ -92,7 +57,7 @@ async fn main() -> Result<()> {
Ok(())
}
async fn download_and_parse_excel(
async fn download_and_parse_excel_in_memory(
client: &mut SynologyClient,
config: &Config,
) -> Result<ExcelWorkbook> {
@@ -145,16 +110,166 @@ async fn download_and_parse_excel(
.ok_or_else(|| anyhow!("Missing name"))?;
info!("Found file: {}", actual_file_name);
info!("Exporting file from Synology Office to Excel format...");
info!("Exporting file from Synology Office to Excel format (in-memory)...");
let output_path = format!("./{}_export.xlsx", search_name);
let data = client.export_by_file_id(file_id, actual_file_name).await?;
let mut file = File::create(&output_path)?;
file.write_all(&data)?;
info!("Exported to: {} ({} bytes)", output_path, data.len());
// Получаем бинарные данные Excel напрямую в память
let excel_data = client.export_by_file_id(file_id, actual_file_name).await?;
info!("Exported {} bytes to memory", excel_data.len());
info!("Parsing Excel workbook...");
read_excel_workbook(&output_path)
info!("Parsing Excel workbook from memory...");
parse_excel_from_bytes(&excel_data)
}
fn parse_excel_from_bytes(data: &[u8]) -> Result<ExcelWorkbook> {
let cursor = Cursor::new(data);
let mut workbook: Xlsx<_> = calamine::open_workbook_from_rs(cursor)
.map_err(|e| anyhow!("Failed to open workbook from memory: {}", e))?;
let sheet_names = workbook.sheet_names().to_vec();
let mut excel_workbook = ExcelWorkbook::default();
for sheet_name in sheet_names {
debug!("Processing sheet: '{}'", sheet_name);
let range = workbook
.worksheet_range(&sheet_name)
.map_err(|e| anyhow!("Failed to read sheet '{}': {}", sheet_name, e))?;
let sheet_data = parse_sheet_dynamic_optimized(&sheet_name, &range)?;
excel_workbook.sheets.push(sheet_data);
}
Ok(excel_workbook)
}
fn parse_sheet_dynamic_optimized(
sheet_name: &str,
range: &calamine::Range<Data>,
) -> Result<SheetData> {
// Предварительно выделяем память для избежания реаллокаций
let (row_count, col_count) = range.get_size();
let mut data_matrix: Vec<Vec<String>> = Vec::with_capacity(row_count);
// Строим матрицу данных (без enumerate)
for row in range.rows() {
let mut row_data = Vec::with_capacity(col_count);
for cell in row {
row_data.push(cell_to_string_optimized(cell));
}
data_matrix.push(row_data);
}
if data_matrix.is_empty() {
return Ok(SheetData {
name: sheet_name.to_string(),
headers: Vec::new(),
rows: Vec::new(),
});
}
// Извлекаем заголовки с дедупликацией
let headers: Vec<String> = data_matrix[0]
.iter()
.enumerate()
.map(|(idx, header)| {
let h = header.trim().to_string();
if h.is_empty() {
format!("Column_{}", idx + 1)
} else {
h
}
})
.collect();
// Парсим строки данных
let rows_data: Vec<HashMap<String, String>> = data_matrix
.iter()
.skip(1)
.filter_map(|row_values| {
let mut row_map = HashMap::with_capacity(headers.len());
let mut has_data = false;
for (col_idx, value) in row_values.iter().enumerate() {
if col_idx < headers.len() && !value.is_empty() {
row_map.insert(headers[col_idx].clone(), value.clone());
has_data = true;
}
}
if has_data {
Some(row_map)
} else {
None
}
})
.take(10000) // Ограничение для безопасности
.collect();
Ok(SheetData {
name: sheet_name.to_string(),
headers,
rows: rows_data,
})
}
#[inline]
fn cell_to_string_optimized(cell: &Data) -> String {
match cell {
Data::Empty => String::new(),
Data::String(s) => s.clone(),
Data::Float(f) => {
if is_excel_date(*f) {
excel_date_to_string(*f)
} else if f.fract() == 0.0 {
// Используем itoa для целых чисел (опционально)
format!("{:.0}", f)
} else {
// Используем ryu для float (опционально)
f.to_string()
}
}
Data::Int(i) => {
let f = *i as f64;
if is_excel_date(f) {
excel_date_to_string(f)
} else {
i.to_string()
}
}
Data::Bool(b) => {
if *b {
"TRUE".to_string()
} else {
"FALSE".to_string()
}
}
Data::DateTime(dt) => {
let serial = dt.as_f64();
if is_excel_date(serial) {
excel_date_to_string(serial)
} else {
dt.to_string()
}
}
Data::DateTimeIso(s) => s.clone(),
Data::DurationIso(s) => s.clone(),
Data::Error(e) => format!("{:?}", e),
}
}
#[inline]
fn is_excel_date(value: f64) -> bool {
(1.0..100000.0).contains(&value)
}
fn excel_date_to_string(serial: f64) -> String {
let days = serial as i64;
let base = NaiveDate::from_ymd_opt(1899, 12, 30).unwrap();
if let Some(date) = base.checked_add_signed(Duration::days(days)) {
// Используем метод format напрямую - он публичный
date.format("%d.%m.%Y").to_string()
} else {
serial.to_string()
}
}
fn display_workbook_structure(workbook: &ExcelWorkbook) {
@@ -168,38 +283,9 @@ fn display_workbook_structure(workbook: &ExcelWorkbook) {
sheet.headers.len(),
sheet.rows.len()
);
if log::log_enabled!(log::Level::Debug) {
debug!(" Headers: {:?}", sheet.headers);
}
}
}
fn save_workbook_json(workbook: &ExcelWorkbook, nas_file: &str) -> Result<()> {
let search_name = Path::new(nas_file)
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("unknown")
.replace(".osheet", "");
let json_output = format!("./{}_workbook.json", search_name);
let json_data = workbook.to_json();
std::fs::write(&json_output, serde_json::to_string_pretty(&json_data)?)?;
info!("Full workbook saved to: {}", json_output);
for sheet in &workbook.sheets {
let safe_name = sheet
.name
.replace(['/', '\\', ':', '*', '?', '"', '<', '>', '|'], "_");
let sheet_json_path = format!("./{}_{}.json", search_name, safe_name);
std::fs::write(
&sheet_json_path,
serde_json::to_string_pretty(&sheet.to_json())?,
)?;
}
Ok(())
}
async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Result<()> {
info!("Preparing Nexrender jobs...");
@@ -216,6 +302,7 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re
.get_sheet("CHANELL")
.ok_or_else(|| anyhow!("Sheet 'CHANELL' not found"))?;
// Используем предварительное выделение памяти
let packs: HashMap<String, String> = sport_sheet
.rows
.iter()
@@ -223,7 +310,7 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re
.collect();
info!("Loaded {} sport packs", packs.len());
let mut logos = LogoRegistry::new();
let mut logos = LogoRegistry::with_capacity(teams_sheet.rows.len());
for row in &teams_sheet.rows {
if let (Some(team), Some(sport), Some(link)) =
(row.get("TEAM"), row.get("SPORT"), row.get("LINK"))
@@ -240,12 +327,13 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re
.collect();
info!("Loaded {} channel logos", channels.len());
let mut jobs: Vec<JobData> = Vec::new();
// Предварительно выделяем память для jobs
let mut jobs: Vec<JobData> = Vec::with_capacity(start_sheet.rows.len() * 3);
for (idx, row) in start_sheet.rows.iter().enumerate() {
if let Some(state) = row.get("STATE") {
if state == "FALSE" {
if let Some(job) = JobData::from_row(row, idx, &packs, &logos, &channels) {
debug!("Created job for row {}: {}", idx, job.outfile_name);
jobs.extend(job.create_variants());
}
}
@@ -262,28 +350,48 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re
cleanup_finished_jobs(&config.nexrender_api_url).await?;
let http_client = Client::new();
let mut submitted_jobs: Vec<(String, String)> = Vec::new();
let mut submitted_jobs: Vec<(String, String)> = Vec::with_capacity(jobs.len());
for job in &jobs {
let nexrender_job = job.to_nexrender_job(&config.output_folder);
info!("Submitting job: {}", job.outfile_name);
// Отправляем jobs параллельно для ускорения
let mut tasks = Vec::with_capacity(jobs.len());
for job in jobs {
let client = http_client.clone();
let api_url = config.nexrender_api_url.clone();
let output_folder = config.output_folder.clone();
let response = http_client
.post(&config.nexrender_api_url)
.json(&nexrender_job)
.send()
.await?;
tasks.push(tokio::spawn(async move {
let nexrender_job = job.to_nexrender_job(&output_folder);
let response = client.post(&api_url).json(&nexrender_job).send().await?;
if response.status().is_success() {
let result: Value = response.json().await?;
if let Some(uid) = result.get("uid").and_then(|u| u.as_str()) {
submitted_jobs.push((uid.to_string(), job.outfile_name.clone()));
info!("Job submitted successfully, UID: {}", uid);
if response.status().is_success() {
let result: Value = response.json().await?;
if let Some(uid) = result.get("uid").and_then(|u| u.as_str()) {
Ok::<_, anyhow::Error>(Some((uid.to_string(), job.outfile_name)))
} else {
Ok(None)
}
} else {
error!("Failed to submit job: {}", job.outfile_name);
Ok(None)
}
}));
}
for task in tasks {
match task.await {
Ok(Ok(Some((uid, outfile_name)))) => {
info!("Job submitted: {}", outfile_name);
submitted_jobs.push((uid, outfile_name));
}
Ok(Ok(None)) => {
debug!("Job submission returned no UID");
}
Ok(Err(e)) => {
error!("Job submission error: {}", e);
}
Err(e) => {
error!("Task join error: {}", e);
}
} else {
let status = response.status();
let text = response.text().await.unwrap_or_default();
error!("Failed to submit job ({}): {}", status, text);
}
}
@@ -300,145 +408,10 @@ async fn process_nexrender_jobs(workbook: &ExcelWorkbook, config: &Config) -> Re
Ok(())
}
fn read_excel_workbook(file_path: &str) -> Result<ExcelWorkbook> {
let mut workbook: Xlsx<_> = calamine::open_workbook(file_path)
.map_err(|e| anyhow!("Failed to open workbook: {}", e))?;
let sheet_names = workbook.sheet_names().to_vec();
let mut excel_workbook = ExcelWorkbook::default();
for sheet_name in sheet_names {
debug!("Processing sheet: '{}'", sheet_name);
let range = workbook
.worksheet_range(&sheet_name)
.map_err(|e| anyhow!("Failed to read sheet '{}': {}", sheet_name, e))?;
let sheet_data = parse_sheet_dynamic(&sheet_name, &range)?;
excel_workbook.sheets.push(sheet_data);
}
Ok(excel_workbook)
}
fn parse_sheet_dynamic(sheet_name: &str, range: &calamine::Range<Data>) -> Result<SheetData> {
let mut cell_map: HashMap<(usize, usize), String> = HashMap::new();
let mut max_row = 0usize;
let mut max_col = 0usize;
for (row, col, cell) in range.cells() {
let value = cell_to_string(cell);
if !value.is_empty() {
cell_map.insert((row, col), value);
max_row = max_row.max(row);
max_col = max_col.max(col);
}
}
if cell_map.is_empty() {
return Ok(SheetData {
name: sheet_name.to_string(),
headers: Vec::new(),
rows: Vec::new(),
});
}
let mut data_matrix: Vec<Vec<String>> = Vec::new();
for row in 0..=max_row {
let mut row_data = Vec::new();
for col in 0..=max_col {
row_data.push(cell_map.get(&(row, col)).cloned().unwrap_or_default());
}
data_matrix.push(row_data);
}
let headers: Vec<String> = if !data_matrix.is_empty() {
data_matrix[0]
.iter()
.enumerate()
.map(|(idx, header)| {
let h = header.trim().to_string();
if h.is_empty() {
format!("Column_{}", idx + 1)
} else {
h
}
})
.collect()
} else {
Vec::new()
};
let mut rows_data = Vec::new();
for row_values in data_matrix.iter().skip(1) {
let mut row_map = HashMap::new();
for (col_idx, value) in row_values.iter().enumerate() {
if col_idx < headers.len() && !value.is_empty() {
row_map.insert(headers[col_idx].clone(), value.clone());
}
}
if !row_map.is_empty() {
rows_data.push(row_map);
}
if rows_data.len() >= 10000 {
break;
}
}
Ok(SheetData {
name: sheet_name.to_string(),
headers,
rows: rows_data,
})
}
fn excel_date_to_string(serial: f64) -> String {
let days = serial as i64;
let base = NaiveDate::from_ymd_opt(1899, 12, 30).unwrap();
if let Some(date) = base.checked_add_signed(Duration::days(days)) {
date.format("%d.%m.%Y").to_string()
} else {
serial.to_string()
}
}
fn is_excel_date(value: f64) -> bool {
(1.0..100000.0).contains(&value)
}
fn cell_to_string(cell: &Data) -> String {
match cell {
Data::Empty => String::new(),
Data::String(s) => s.clone(),
Data::Float(f) => {
if is_excel_date(*f) {
excel_date_to_string(*f)
} else if f.fract() == 0.0 {
format!("{:.0}", f)
} else {
f.to_string()
}
}
Data::Int(i) => {
let f = *i as f64;
if is_excel_date(f) {
excel_date_to_string(f)
} else {
i.to_string()
}
}
Data::Bool(b) => b.to_string(),
Data::DateTime(dt) => {
let serial = dt.as_f64();
if is_excel_date(serial) {
excel_date_to_string(serial)
} else {
dt.to_string()
}
}
Data::DateTimeIso(s) => s.clone(),
Data::DurationIso(s) => s.clone(),
Data::Error(e) => format!("{:?}", e),
// ... остальные функции без изменений ...
impl ExcelWorkbook {
pub fn get_sheet(&self, name: &str) -> Option<&SheetData> {
self.sheets.iter().find(|s| s.name == name)
}
}
@@ -454,10 +427,7 @@ async fn cleanup_finished_jobs(api_url: &str) -> Result<()> {
job.get("state").and_then(|s| s.as_str()),
) {
if state == "finished" || state == "error" {
client
.delete(&format!("{}/{}", api_url, uid))
.send()
.await?;
let _ = client.delete(&format!("{}/{}", api_url, uid)).send().await;
info!("Cleaned up completed job: {}", uid);
}
}
@@ -474,8 +444,8 @@ async fn monitor_jobs(
while !pending.is_empty() {
sleep(StdDuration::from_secs(25)).await;
let mut remaining = Vec::new();
for (uid, outname) in &pending {
let mut remaining = Vec::with_capacity(pending.len());
for (uid, outname) in pending.drain(..) {
let response = client.get(&format!("{}/{}", api_url, uid)).send().await?;
if response.status().is_success() {
@@ -487,17 +457,13 @@ async fn monitor_jobs(
match state {
"finished" => {
info!(
"Job completed: {} ({} remaining)",
outname,
pending.len() - 1
)
info!("Job completed: {}", outname)
}
"error" => error!("Job failed: {} ({} remaining)", outname, pending.len() - 1),
_ => remaining.push((uid.clone(), outname.clone())),
"error" => error!("Job failed: {}", outname),
_ => remaining.push((uid, outname)),
}
} else {
remaining.push((uid.clone(), outname.clone()));
remaining.push((uid, outname));
}
}
pending = remaining;

6
src/nexrender.rs
View File

@@ -507,6 +507,12 @@ impl LogoRegistry {
self.logos.insert((team, sport), link);
}
pub fn with_capacity(capacity: usize) -> Self {
Self {
logos: HashMap::with_capacity(capacity),
}
}
pub fn find(&self, team: &str, sport: &str) -> Option<String> {
if let Some(link) = self.logos.get(&(team.to_string(), sport.to_string())) {
return Some(link.clone());