Алексей Барабанов
9b3d959ce3
- Добавлена сортировка заданий по дате и времени события (самые ранние — первыми) - Добавлена сортировка по варианту (оригинал -> today -> tomorrow) - Изменена отправка на последовательную для гарантированного порядка в очереди - Исправлена нормализация дефисов (2+ дефиса -> 1) - Исправлен SINGLE шаблон (убран слой TEAMS) - Добавлена поддержка cache: true/false для ассетов - Логотипы команд: cache: false (часто меняются) - Логотип канала: cache: true (редко меняется) - Видео-пак: cache: true (редко меняется) - Версия 0.2.6
561 lines
18 KiB
Rust
561 lines
18 KiB
Rust
use crate::config::Config;
|
||
use crate::nexrender::{JobData, LogoRegistry};
|
||
use crate::synology::SynologyClient;
|
||
use anyhow::{anyhow, Result};
|
||
use calamine::{Data, Reader, Xlsx};
|
||
use chrono::{Duration, NaiveDate};
|
||
use log::{debug, error, info, log_enabled, Level};
|
||
use reqwest::Client;
|
||
use serde_json::Value;
|
||
use std::collections::{HashMap, HashSet};
|
||
use std::io::Cursor;
|
||
use std::path::Path;
|
||
|
||
#[derive(Debug, Clone)]
|
||
pub struct SheetData {
|
||
pub name: String,
|
||
pub headers: Vec<String>,
|
||
pub rows: Vec<HashMap<String, String>>,
|
||
}
|
||
|
||
#[derive(Debug, Default)]
|
||
pub struct ExcelWorkbook {
|
||
pub sheets: Vec<SheetData>,
|
||
}
|
||
|
||
impl ExcelWorkbook {
|
||
pub fn get_sheet(&self, name: &str) -> Option<&SheetData> {
|
||
self.sheets.iter().find(|s| s.name == name)
|
||
}
|
||
}
|
||
|
||
/// Основная функция обработки: скачивает Excel, генерирует задания, отправляет в Nexrender
|
||
pub async fn process_spreadsheet(config: &Config) -> Result<Vec<(String, String)>> {
|
||
let mut client = SynologyClient::new(&config.nas_fqdn);
|
||
client.login(&config.nas_user, &config.nas_pass).await?;
|
||
info!("Successfully authenticated with Synology NAS");
|
||
|
||
let info = client.get_info().await?;
|
||
info!("Connected to NAS: {}", info.hostname);
|
||
|
||
// Скачиваем и парсим Excel в памяти
|
||
let workbook = download_and_parse_excel_in_memory(&mut client, config).await?;
|
||
display_workbook_structure(&workbook);
|
||
|
||
let submitted = generate_and_submit_jobs(&workbook, config).await?;
|
||
|
||
client.logout().await?;
|
||
info!("Session terminated successfully");
|
||
|
||
Ok(submitted)
|
||
}
|
||
|
||
async fn download_and_parse_excel_in_memory(
|
||
client: &mut SynologyClient,
|
||
config: &Config,
|
||
) -> Result<ExcelWorkbook> {
|
||
let file_name_full = Path::new(&config.nas_file)
|
||
.file_name()
|
||
.and_then(|n| n.to_str())
|
||
.unwrap_or("unknown");
|
||
let search_name = file_name_full.replace(".osheet", "");
|
||
let expected_path = Path::new(&config.nas_file)
|
||
.parent()
|
||
.and_then(|p| p.to_str())
|
||
.unwrap_or("");
|
||
|
||
let api_format_path = expected_path
|
||
.replace("Team Folder", "team-folders")
|
||
.replace(' ', "");
|
||
|
||
let full_expected_path_api = format!("{}/{}", api_format_path, file_name_full);
|
||
info!("Searching for file: {}", full_expected_path_api);
|
||
|
||
let search_result = client.search_file_by_name(&search_name).await?;
|
||
let items = search_result["data"]["items"]
|
||
.as_array()
|
||
.ok_or_else(|| anyhow!("No items in search result"))?;
|
||
|
||
if items.is_empty() {
|
||
return Err(anyhow!("File '{}' not found", file_name_full));
|
||
}
|
||
|
||
let exact_match: Vec<&Value> = items
|
||
.iter()
|
||
.filter(|item| {
|
||
item.get("display_path")
|
||
.and_then(|p| p.as_str())
|
||
.unwrap_or("")
|
||
== full_expected_path_api
|
||
})
|
||
.collect();
|
||
|
||
if exact_match.is_empty() {
|
||
return Err(anyhow!("File not found at: {}", full_expected_path_api));
|
||
}
|
||
|
||
let file = exact_match[0];
|
||
let file_id = file["file_id"]
|
||
.as_str()
|
||
.ok_or_else(|| anyhow!("Missing file_id"))?;
|
||
let actual_file_name = file["name"]
|
||
.as_str()
|
||
.ok_or_else(|| anyhow!("Missing name"))?;
|
||
|
||
info!("Found file: {}", actual_file_name);
|
||
info!("Exporting file from Synology Office to Excel format (in-memory)...");
|
||
|
||
// Получаем бинарные данные Excel напрямую в память
|
||
let excel_data = client.export_by_file_id(file_id, actual_file_name).await?;
|
||
info!("Exported {} bytes to memory", excel_data.len());
|
||
|
||
info!("Parsing Excel workbook from memory...");
|
||
parse_excel_from_bytes(&excel_data)
|
||
}
|
||
|
||
fn parse_excel_from_bytes(data: &[u8]) -> Result<ExcelWorkbook> {
|
||
let cursor = Cursor::new(data);
|
||
let mut workbook: Xlsx<_> = calamine::open_workbook_from_rs(cursor)
|
||
.map_err(|e| anyhow!("Failed to open workbook from memory: {}", e))?;
|
||
|
||
let sheet_names = workbook.sheet_names().to_vec();
|
||
let mut excel_workbook = ExcelWorkbook::default();
|
||
|
||
for sheet_name in sheet_names {
|
||
debug!("Processing sheet: '{}'", sheet_name);
|
||
|
||
let range = workbook
|
||
.worksheet_range(&sheet_name)
|
||
.map_err(|e| anyhow!("Failed to read sheet '{}': {}", sheet_name, e))?;
|
||
|
||
let sheet_data = parse_sheet_dynamic_optimized(&sheet_name, &range)?;
|
||
excel_workbook.sheets.push(sheet_data);
|
||
}
|
||
|
||
Ok(excel_workbook)
|
||
}
|
||
|
||
fn parse_sheet_dynamic_optimized(
|
||
sheet_name: &str,
|
||
range: &calamine::Range<Data>,
|
||
) -> Result<SheetData> {
|
||
let (row_count, col_count) = range.get_size();
|
||
let mut data_matrix: Vec<Vec<String>> = Vec::with_capacity(row_count);
|
||
|
||
for row in range.rows() {
|
||
let mut row_data = Vec::with_capacity(col_count);
|
||
for cell in row {
|
||
row_data.push(cell_to_string_optimized(cell));
|
||
}
|
||
data_matrix.push(row_data);
|
||
}
|
||
|
||
if data_matrix.is_empty() {
|
||
return Ok(SheetData {
|
||
name: sheet_name.to_string(),
|
||
headers: Vec::new(),
|
||
rows: Vec::new(),
|
||
});
|
||
}
|
||
|
||
// Извлекаем заголовки с дедупликацией
|
||
let headers: Vec<String> = data_matrix[0]
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(idx, header)| {
|
||
let h = header.trim().to_string();
|
||
if h.is_empty() {
|
||
format!("Column_{}", idx + 1)
|
||
} else {
|
||
h
|
||
}
|
||
})
|
||
.collect();
|
||
|
||
let rows_data: Vec<HashMap<String, String>> = data_matrix
|
||
.iter()
|
||
.skip(1)
|
||
.filter_map(|row_values| {
|
||
let mut row_map = HashMap::with_capacity(headers.len());
|
||
let mut has_data = false;
|
||
|
||
for (col_idx, value) in row_values.iter().enumerate() {
|
||
if col_idx < headers.len() && !value.is_empty() {
|
||
row_map.insert(headers[col_idx].clone(), value.clone());
|
||
has_data = true;
|
||
}
|
||
}
|
||
|
||
if has_data {
|
||
Some(row_map)
|
||
} else {
|
||
None
|
||
}
|
||
})
|
||
.take(10000) // Ограничение для безопасности
|
||
.collect();
|
||
|
||
Ok(SheetData {
|
||
name: sheet_name.to_string(),
|
||
headers,
|
||
rows: rows_data,
|
||
})
|
||
}
|
||
|
||
#[inline]
|
||
fn cell_to_string_optimized(cell: &Data) -> String {
|
||
match cell {
|
||
Data::Empty => String::new(),
|
||
Data::String(s) => s.clone(),
|
||
Data::Float(f) => {
|
||
if is_excel_date(*f) {
|
||
excel_date_to_string(*f)
|
||
} else if f.fract() == 0.0 {
|
||
format!("{:.0}", f)
|
||
} else {
|
||
f.to_string()
|
||
}
|
||
}
|
||
Data::Int(i) => {
|
||
let f = *i as f64;
|
||
if is_excel_date(f) {
|
||
excel_date_to_string(f)
|
||
} else {
|
||
i.to_string()
|
||
}
|
||
}
|
||
Data::Bool(b) => {
|
||
if *b {
|
||
"TRUE".to_string()
|
||
} else {
|
||
"FALSE".to_string()
|
||
}
|
||
}
|
||
Data::DateTime(dt) => {
|
||
let serial = dt.as_f64();
|
||
if is_excel_date(serial) {
|
||
excel_date_to_string(serial)
|
||
} else {
|
||
dt.to_string()
|
||
}
|
||
}
|
||
Data::DateTimeIso(s) => s.clone(),
|
||
Data::DurationIso(s) => s.clone(),
|
||
Data::Error(e) => format!("{:?}", e),
|
||
}
|
||
}
|
||
|
||
#[inline]
|
||
fn is_excel_date(value: f64) -> bool {
|
||
(1.0..100000.0).contains(&value)
|
||
}
|
||
|
||
fn excel_date_to_string(serial: f64) -> String {
|
||
let days = serial as i64;
|
||
let base = NaiveDate::from_ymd_opt(1899, 12, 30).unwrap();
|
||
if let Some(date) = base.checked_add_signed(Duration::days(days)) {
|
||
date.format("%d.%m.%Y").to_string()
|
||
} else {
|
||
serial.to_string()
|
||
}
|
||
}
|
||
|
||
fn display_workbook_structure(workbook: &ExcelWorkbook) {
|
||
info!("Workbook contains {} sheets", workbook.sheets.len());
|
||
|
||
for (idx, sheet) in workbook.sheets.iter().enumerate() {
|
||
debug!(
|
||
"Sheet #{}: '{}' | Headers: {} | Rows: {}",
|
||
idx + 1,
|
||
sheet.name,
|
||
sheet.headers.len(),
|
||
sheet.rows.len()
|
||
);
|
||
}
|
||
}
|
||
|
||
// Вспомогательная функция для поиска значения по части ключа
|
||
fn get_cell_fuzzy(row: &HashMap<String, String>, key_part: &str) -> Option<String> {
|
||
if let Some(val) = row.get(key_part) {
|
||
if !val.is_empty() {
|
||
return Some(val.clone());
|
||
}
|
||
}
|
||
for (k, v) in row.iter() {
|
||
if k.to_lowercase().contains(&key_part.to_lowercase()) && !v.is_empty() {
|
||
return Some(v.clone());
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
async fn generate_and_submit_jobs(
|
||
workbook: &ExcelWorkbook,
|
||
config: &Config,
|
||
) -> Result<Vec<(String, String)>> {
|
||
info!("Preparing Nexrender jobs...");
|
||
|
||
// ========== ЗАГРУЖАЕМ SPORT PACKS ==========
|
||
let sport_sheet = workbook
|
||
.get_sheet("SPORT")
|
||
.ok_or_else(|| anyhow!("Sheet 'SPORT' not found"))?;
|
||
|
||
let packs: HashMap<String, String> = sport_sheet
|
||
.rows
|
||
.iter()
|
||
.filter_map(|row| Some((row.get("SPORT")?.clone(), row.get("LINK")?.clone())))
|
||
.collect();
|
||
info!("Loaded {} sport packs", packs.len());
|
||
debug!("Processed SPORT sheet ({} rows)", sport_sheet.rows.len());
|
||
|
||
// ========== ЗАГРУЖАЕМ TEAM LOGOS ==========
|
||
let teams_sheet = workbook
|
||
.get_sheet("TEAMS")
|
||
.ok_or_else(|| anyhow!("Sheet 'TEAMS' not found"))?;
|
||
|
||
debug!("TEAMS headers: {:?}", teams_sheet.headers);
|
||
if log_enabled!(Level::Debug) {
|
||
for (i, row) in teams_sheet.rows.iter().take(3).enumerate() {
|
||
debug!("TEAMS row {}: {:?}", i, row);
|
||
}
|
||
}
|
||
|
||
let mut logos = LogoRegistry::with_capacity(teams_sheet.rows.len());
|
||
for row in &teams_sheet.rows {
|
||
let team = get_cell_fuzzy(row, "TEAM");
|
||
let sport = get_cell_fuzzy(row, "SPORT");
|
||
let link = get_cell_fuzzy(row, "LINK");
|
||
|
||
if let (Some(team), Some(sport), Some(link)) = (team, sport, link) {
|
||
logos.insert(team.clone(), sport.clone(), link.clone());
|
||
}
|
||
}
|
||
let total_teams = teams_sheet.rows.len();
|
||
info!("Loaded {} team logos", total_teams);
|
||
|
||
// Дебаг: выводим статистику по TEAMS
|
||
if log_enabled!(Level::Debug) {
|
||
let sports: HashSet<_> = teams_sheet
|
||
.rows
|
||
.iter()
|
||
.filter_map(|r| r.get("SPORT"))
|
||
.collect();
|
||
debug!("Sports found in TEAMS sheet: {:?}", sports);
|
||
|
||
for sport in sports.iter().take(5) {
|
||
let examples: Vec<_> = teams_sheet
|
||
.rows
|
||
.iter()
|
||
.filter(|r| r.get("SPORT") == Some(*sport))
|
||
.take(3)
|
||
.filter_map(|r| r.get("TEAM"))
|
||
.collect();
|
||
debug!(" TEAMS examples for '{}': {:?}", sport, examples);
|
||
}
|
||
}
|
||
|
||
// ========== ЗАГРУЖАЕМ CHANNEL LOGOS ==========
|
||
let channel_sheet = workbook
|
||
.get_sheet("CHANELL")
|
||
.ok_or_else(|| anyhow!("Sheet 'CHANELL' not found"))?;
|
||
|
||
let channels: HashMap<String, String> = channel_sheet
|
||
.rows
|
||
.iter()
|
||
.filter_map(|row| Some((row.get("CHANELL")?.clone(), row.get("LINK")?.clone())))
|
||
.collect();
|
||
info!("Loaded {} channel logos", channels.len());
|
||
|
||
if log_enabled!(Level::Debug) {
|
||
debug!("Channels loaded: {:?}", channels.keys().collect::<Vec<_>>());
|
||
}
|
||
|
||
// ========== ОБРАБАТЫВАЕМ START (только активные строки) ==========
|
||
let start_sheet = workbook
|
||
.get_sheet("Start")
|
||
.ok_or_else(|| anyhow!("Sheet 'Start' not found"))?;
|
||
|
||
let total_start_rows = start_sheet.rows.len();
|
||
|
||
// Сразу фильтруем только строки с STATE = "FALSE"
|
||
let active_rows: Vec<(usize, &HashMap<String, String>)> = start_sheet
|
||
.rows
|
||
.iter()
|
||
.enumerate()
|
||
.filter(|(_, row)| row.get("STATE").map(|s| s.as_str()) == Some("FALSE"))
|
||
.collect();
|
||
|
||
info!(
|
||
"Found {} active rows (STATE='FALSE') out of {} total",
|
||
active_rows.len(),
|
||
total_start_rows
|
||
);
|
||
|
||
// Собираем использованные команды для очистки logos
|
||
let mut used_teams: HashSet<(String, String)> = HashSet::new();
|
||
let mut jobs: Vec<JobData> = Vec::with_capacity(active_rows.len() * 3);
|
||
|
||
// Для дебага
|
||
let mut missing_teams: HashSet<String> = HashSet::new();
|
||
let mut missing_sports: HashSet<String> = HashSet::new();
|
||
|
||
for (idx, row) in active_rows {
|
||
let team_a = row.get("TEAM A").cloned().unwrap_or_default();
|
||
let team_b = row.get("TEAM B").cloned().unwrap_or_default();
|
||
let sport = row.get("SPORT").cloned().unwrap_or_default();
|
||
|
||
// Запоминаем использованные команды
|
||
if !team_a.is_empty() {
|
||
used_teams.insert((team_a.clone(), sport.clone()));
|
||
}
|
||
if !team_b.is_empty() {
|
||
used_teams.insert((team_b.clone(), sport.clone()));
|
||
}
|
||
|
||
if log_enabled!(Level::Debug) {
|
||
let logo_a = logos.find(&team_a, &sport);
|
||
let logo_b = logos.find(&team_b, &sport);
|
||
|
||
if !team_a.is_empty() && logo_a.is_none() {
|
||
missing_teams.insert(team_a.clone());
|
||
missing_sports.insert(sport.clone());
|
||
}
|
||
if !team_b.is_empty() && logo_b.is_none() {
|
||
missing_teams.insert(team_b.clone());
|
||
missing_sports.insert(sport.clone());
|
||
}
|
||
}
|
||
|
||
if let Some(job) = JobData::from_row(row, idx, &packs, &logos, &channels) {
|
||
jobs.extend(job.create_variants());
|
||
}
|
||
}
|
||
|
||
// Очищаем logos от неиспользуемых команд
|
||
let logos_before = logos.len();
|
||
logos.retain(|team, sport| used_teams.contains(&(team.to_string(), sport.to_string())));
|
||
info!(
|
||
"Retained {} used team logos (cleaned up {} unused)",
|
||
logos.len(),
|
||
logos_before - logos.len()
|
||
);
|
||
|
||
// Дебаг: выводим сводку по отсутствующим логотипам
|
||
if log_enabled!(Level::Debug) && !missing_teams.is_empty() {
|
||
debug!("=== MISSING LOGOS SUMMARY ===");
|
||
debug!("Total unique teams missing logos: {}", missing_teams.len());
|
||
debug!("Sports involved: {:?}", missing_sports);
|
||
debug!(
|
||
"Missing teams (first 20): {:?}",
|
||
missing_teams.iter().take(20).collect::<Vec<_>>()
|
||
);
|
||
debug!("==============================");
|
||
}
|
||
|
||
info!("Generated {} total jobs (including variants)", jobs.len());
|
||
|
||
// ========== СОРТИРОВКА ПО ДАТЕ И ВРЕМЕНИ ==========
|
||
jobs.sort_by(|a, b| match a.sort_date.cmp(&b.sort_date) {
|
||
std::cmp::Ordering::Equal => match a.sort_time.cmp(&b.sort_time) {
|
||
std::cmp::Ordering::Equal => a.variant_order.cmp(&b.variant_order),
|
||
other => other,
|
||
},
|
||
other => other,
|
||
});
|
||
|
||
info!("Jobs sorted by date and time (earliest first)");
|
||
|
||
if log_enabled!(Level::Debug) {
|
||
for (i, job) in jobs.iter().take(10).enumerate() {
|
||
debug!(
|
||
" {}: {} {} - {} (variant: {})",
|
||
i + 1,
|
||
job.sort_date,
|
||
job.sort_time,
|
||
job.outfile_name,
|
||
job.variant_order
|
||
);
|
||
}
|
||
}
|
||
|
||
if jobs.is_empty() {
|
||
info!("No jobs with STATE='FALSE' found");
|
||
return Ok(Vec::new());
|
||
}
|
||
|
||
// ========== ОТПРАВКА ЗАДАНИЙ (ПОСЛЕДОВАТЕЛЬНО) ==========
|
||
cleanup_finished_jobs(&config.nexrender_api_url).await?;
|
||
|
||
let http_client = Client::new();
|
||
let mut submitted_jobs: Vec<(String, String)> = Vec::with_capacity(jobs.len());
|
||
|
||
for job in jobs {
|
||
let nexrender_job = job.to_nexrender_job(config);
|
||
info!("Submitting job: {}", job.outfile_name);
|
||
|
||
if log_enabled!(Level::Debug) {
|
||
debug!(
|
||
"Job details: sport='{}', league='{}', channel='{}', team_a='{}', team_b='{}'",
|
||
job.sport, job.league, job.channel, job.team_a, job.team_b
|
||
);
|
||
}
|
||
|
||
let response = http_client
|
||
.post(&config.nexrender_api_url)
|
||
.json(&nexrender_job)
|
||
.send()
|
||
.await?;
|
||
|
||
if response.status().is_success() {
|
||
let result: Value = response.json().await?;
|
||
if let Some(uid) = result.get("uid").and_then(|u| u.as_str()) {
|
||
info!("Job submitted: {} (UID: {})", job.outfile_name, uid);
|
||
submitted_jobs.push((uid.to_string(), job.outfile_name.clone()));
|
||
}
|
||
} else {
|
||
let status = response.status();
|
||
let text = response.text().await.unwrap_or_default();
|
||
error!("Failed to submit job ({}): {}", status, text);
|
||
}
|
||
}
|
||
|
||
info!(
|
||
"Successfully submitted {} jobs to Nexrender",
|
||
submitted_jobs.len()
|
||
);
|
||
|
||
Ok(submitted_jobs)
|
||
}
|
||
|
||
pub async fn cleanup_finished_jobs(api_url: &str) -> Result<()> {
|
||
let client = Client::new();
|
||
let response = client.get(api_url).send().await?;
|
||
|
||
if response.status().is_success() {
|
||
let jobs: Vec<Value> = response.json().await?;
|
||
for job in jobs {
|
||
if let (Some(uid), Some(state)) = (
|
||
job.get("uid").and_then(|u| u.as_str()),
|
||
job.get("state").and_then(|s| s.as_str()),
|
||
) {
|
||
if state == "finished" || state == "error" {
|
||
let _ = client.delete(&format!("{}/{}", api_url, uid)).send().await;
|
||
info!("Cleaned up completed job: {}", uid);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
pub async fn fetch_all_jobs(api_url: &str) -> Result<Vec<Value>> {
|
||
let client = Client::new();
|
||
let response = client.get(api_url).send().await?;
|
||
if response.status().is_success() {
|
||
let jobs: Vec<Value> = response.json().await?;
|
||
Ok(jobs)
|
||
} else {
|
||
Ok(Vec::new())
|
||
}
|
||
}
|