lexx/AE_Anons
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases 3 Activity

Оптимизирована работа с памятью

Browse Source
This commit is contained in:
Алексей Барабанов
2026-04-17 19:14:38 +03:00
parent c3056010eb
commit e06b55ad07
7 changed files with 184 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/nexrender.rs
View File

@@ -564,6 +564,20 @@ impl LogoRegistry {
log::warn!("No logo found for team='{}', sport='{}'", team, sport);
None
}
/// Оставляет только указанные команды, удаляя все остальные
pub fn retain<F>(&mut self, mut predicate: F)
where
F: FnMut(&str, &str) -> bool,
{
self.logos.retain(|(team, sport), _| predicate(team, sport));
}
/// Возвращает количество записей в реестре
pub fn len(&self) -> usize {
self.logos.len()
}
}
fn parse_date(s: &str) -> (NaiveDate, String) {

294
src/processor.rs
View File

@@ -274,54 +274,56 @@ fn display_workbook_structure(workbook: &ExcelWorkbook) {
}
}
// Вспомогательная функция для поиска значения по части ключа
fn get_cell_fuzzy(row: &HashMap<String, String>, key_part: &str) -> Option<String> {
if let Some(val) = row.get(key_part) {
if !val.is_empty() {
return Some(val.clone());
}
}
for (k, v) in row.iter() {
if k.to_lowercase().contains(&key_part.to_lowercase()) && !v.is_empty() {
return Some(v.clone());
}
}
None
}
async fn generate_and_submit_jobs(
workbook: &ExcelWorkbook,
config: &Config,
) -> Result<Vec<(String, String)>> {
info!("Preparing Nexrender jobs...");
let start_sheet = workbook
.get_sheet("Start")
.ok_or_else(|| anyhow!("Sheet 'Start' not found"))?;
let sport_sheet = workbook
.get_sheet("SPORT")
.ok_or_else(|| anyhow!("Sheet 'SPORT' not found"))?;
// ========== ЗАГРУЖАЕМ SPORT PACKS ==========
let sport_sheet = workbook
.get_sheet("SPORT")
.ok_or_else(|| anyhow!("Sheet 'SPORT' not found"))?;
let packs: HashMap<String, String> = sport_sheet
.rows
.iter()
.filter_map(|row| Some((row.get("SPORT")?.clone(), row.get("LINK")?.clone())))
.collect();
info!("Loaded {} sport packs", packs.len());
// sport_sheet автоматически освободится при выходе из функции
// Лист SPORT больше не нужен
let sport_rows_count = sport_sheet.rows.len();
debug!("Dropped SPORT sheet ({} rows)", sport_rows_count);
// ========== ЗАГРУЖАЕМ TEAM LOGOS ==========
let teams_sheet = workbook
.get_sheet("TEAMS")
.ok_or_else(|| anyhow!("Sheet 'TEAMS' not found"))?;
let channel_sheet = workbook
.get_sheet("CHANELL")
.ok_or_else(|| anyhow!("Sheet 'CHANELL' not found"))?;
// Загружаем sport packs
let packs: HashMap<String, String> = sport_sheet
.rows
.iter()
.filter_map(|row| Some((row.get("SPORT")?.clone(), row.get("LINK")?.clone())))
.collect();
info!("Loaded {} sport packs", packs.len());
// Вспомогательная функция для поиска значения по части ключа
fn get_cell_fuzzy(row: &HashMap<String, String>, key_part: &str) -> Option<String> {
if let Some(val) = row.get(key_part) {
if !val.is_empty() {
return Some(val.clone());
}
}
for (k, v) in row.iter() {
if k.to_lowercase().contains(&key_part.to_lowercase()) && !v.is_empty() {
return Some(v.clone());
}
}
None
}
// Загружаем team logos
debug!("TEAMS headers: {:?}", teams_sheet.headers);
for (i, row) in teams_sheet.rows.iter().take(3).enumerate() {
debug!("TEAMS row {}: {:?}", i, row);
if log_enabled!(Level::Debug) {
for (i, row) in teams_sheet.rows.iter().take(3).enumerate() {
debug!("TEAMS row {}: {:?}", i, row);
}
}
let mut logos = LogoRegistry::with_capacity(teams_sheet.rows.len());
for row in &teams_sheet.rows {
let team = get_cell_fuzzy(row, "TEAM");
@@ -332,8 +334,9 @@ async fn generate_and_submit_jobs(
logos.insert(team.clone(), sport.clone(), link.clone());
}
}
info!("Loaded {} team logos", teams_sheet.rows.len());
let total_teams = teams_sheet.rows.len();
info!("Loaded {} team logos", total_teams);
// Дебаг: выводим статистику по TEAMS
if log_enabled!(Level::Debug) {
let sports: HashSet<_> = teams_sheet
@@ -343,63 +346,106 @@ async fn generate_and_submit_jobs(
.collect();
debug!("Sports found in TEAMS sheet: {:?}", sports);
for sport in sports.iter() {
for sport in sports.iter().take(5) {
let examples: Vec<_> = teams_sheet
.rows
.iter()
.filter(|r| r.get("SPORT") == Some(*sport))
.take(5)
.take(3)
.filter_map(|r| r.get("TEAM"))
.collect();
debug!(" TEAMS examples for '{}': {:?}", sport, examples);
}
}
// Загружаем channel logos
// ========== ЗАГРУЖАЕМ CHANNEL LOGOS ==========
let channel_sheet = workbook
.get_sheet("CHANELL")
.ok_or_else(|| anyhow!("Sheet 'CHANELL' not found"))?;
let channels: HashMap<String, String> = channel_sheet
.rows
.iter()
.filter_map(|row| Some((row.get("CHANELL")?.clone(), row.get("LINK")?.clone())))
.collect();
info!("Loaded {} channel logos", channels.len());
if log_enabled!(Level::Debug) {
debug!("Channels loaded: {:?}", channels.keys().collect::<Vec<_>>());
}
let mut jobs: Vec<JobData> = Vec::with_capacity(start_sheet.rows.len() * 3);
// Собираем статистику по ненайденным логотипам для дебага
// ========== ОБРАБАТЫВАЕМ START (только активные строки) ==========
let start_sheet = workbook
.get_sheet("Start")
.ok_or_else(|| anyhow!("Sheet 'Start' not found"))?;
let total_start_rows = start_sheet.rows.len();
// Сразу фильтруем только строки с STATE = "FALSE"
let active_rows: Vec<(usize, &HashMap<String, String>)> = start_sheet
.rows
.iter()
.enumerate()
.filter(|(_, row)| row.get("STATE").map(|s| s.as_str()) == Some("FALSE"))
.collect();
info!(
"Found {} active rows (STATE='FALSE') out of {} total",
active_rows.len(),
total_start_rows
);
// Собираем использованные команды для очистки logos
let mut used_teams: HashSet<(String, String)> = HashSet::new();
let mut jobs: Vec<JobData> = Vec::with_capacity(active_rows.len() * 3);
// Для дебага
let mut missing_teams: HashSet<String> = HashSet::new();
let mut missing_sports: HashSet<String> = HashSet::new();
for (idx, row) in active_rows {
let team_a = row.get("TEAM A").cloned().unwrap_or_default();
let team_b = row.get("TEAM B").cloned().unwrap_or_default();
let sport = row.get("SPORT").cloned().unwrap_or_default();
// Запоминаем использованные команды
if !team_a.is_empty() {
used_teams.insert((team_a.clone(), sport.clone()));
}
if !team_b.is_empty() {
used_teams.insert((team_b.clone(), sport.clone()));
}
if log_enabled!(Level::Debug) {
let logo_a = logos.find(&team_a, &sport);
let logo_b = logos.find(&team_b, &sport);
for (idx, row) in start_sheet.rows.iter().enumerate() {
if let Some(state) = row.get("STATE") {
if state == "FALSE" {
let team_a = row.get("TEAM A").cloned().unwrap_or_default();
let team_b = row.get("TEAM B").cloned().unwrap_or_default();
let sport = row.get("SPORT").cloned().unwrap_or_default();
if log_enabled!(Level::Debug) {
let logo_a = logos.find(&team_a, &sport);
let logo_b = logos.find(&team_b, &sport);
if !team_a.is_empty() && logo_a.is_none() {
missing_teams.insert(team_a.clone());
missing_sports.insert(sport.clone());
}
if !team_b.is_empty() && logo_b.is_none() {
missing_teams.insert(team_b.clone());
missing_sports.insert(sport.clone());
}
}
if let Some(job) = JobData::from_row(row, idx, &packs, &logos, &channels) {
jobs.extend(job.create_variants());
}
if !team_a.is_empty() && logo_a.is_none() {
missing_teams.insert(team_a.clone());
missing_sports.insert(sport.clone());
}
if !team_b.is_empty() && logo_b.is_none() {
missing_teams.insert(team_b.clone());
missing_sports.insert(sport.clone());
}
}
if let Some(job) = JobData::from_row(row, idx, &packs, &logos, &channels) {
jobs.extend(job.create_variants());
}
}
// Очищаем logos от неиспользуемых команд
let logos_before = logos.len();
logos.retain(|team, sport| used_teams.contains(&(team.to_string(), sport.to_string())));
info!(
"Retained {} used team logos (cleaned up {} unused)",
logos.len(),
logos_before - logos.len()
);
// Дебаг: выводим сводку по отсутствующим логотипам
if log_enabled!(Level::Debug) && !missing_teams.is_empty() {
@@ -420,70 +466,70 @@ async fn generate_and_submit_jobs(
return Ok(Vec::new());
}
cleanup_finished_jobs(&config.nexrender_api_url).await?;
// ========== ОТПРАВКА ЗАДАНИЙ ==========
cleanup_finished_jobs(&config.nexrender_api_url).await?;
let http_client = Client::new();
let mut submitted_jobs: Vec<(String, String)> = Vec::with_capacity(jobs.len());
let http_client = Client::new();
let mut submitted_jobs: Vec<(String, String)> = Vec::with_capacity(jobs.len());
// Отправляем задания группами по 3 (оригинал + today + tomorrow)
for chunk in jobs.chunks(3) {
let mut tasks = Vec::with_capacity(3);
// Запускаем ВСЕ группы параллельно
let mut all_tasks = Vec::new();
for chunk in jobs.chunks(3) {
for job in chunk {
let client = http_client.clone();
let api_url = config.nexrender_api_url.clone();
let config_clone = config.clone();
let job_owned = job.clone();
for job in chunk {
let client = http_client.clone();
let api_url = config.nexrender_api_url.clone();
let config_clone = config.clone();
let job_owned = job.clone();
all_tasks.push(tokio::spawn(async move {
let nexrender_job = job_owned.to_nexrender_job(&config_clone);
info!("Submitting job: {}", job_owned.outfile_name);
tasks.push(tokio::spawn(async move {
let nexrender_job = job_owned.to_nexrender_job(&config_clone);
info!("Submitting job: {}", job_owned.outfile_name);
if log_enabled!(Level::Debug) {
debug!(
"Job details: sport='{}', league='{}', channel='{}', team_a='{}', team_b='{}'",
job_owned.sport, job_owned.league, job_owned.channel, job_owned.team_a, job_owned.team_b
);
}
let response = client.post(&api_url).json(&nexrender_job).send().await?;
if response.status().is_success() {
let result: Value = response.json().await?;
if let Some(uid) = result.get("uid").and_then(|u| u.as_str()) {
Ok::<_, anyhow::Error>(Some((uid.to_string(), job_owned.outfile_name)))
} else {
Ok(None)
}
if log_enabled!(Level::Debug) {
debug!(
"Job details: sport='{}', league='{}', channel='{}', team_a='{}', team_b='{}'",
job_owned.sport, job_owned.league, job_owned.channel, job_owned.team_a, job_owned.team_b
);
}
let response = client.post(&api_url).json(&nexrender_job).send().await?;
if response.status().is_success() {
let result: Value = response.json().await?;
if let Some(uid) = result.get("uid").and_then(|u| u.as_str()) {
Ok::<_, anyhow::Error>(Some((uid.to_string(), job_owned.outfile_name)))
} else {
let status = response.status();
let text = response.text().await.unwrap_or_default();
error!("Failed to submit job ({}): {}", status, text);
Ok(None)
}
}));
}
// Ждём завершения группы
for task in tasks {
match task.await {
Ok(Ok(Some((uid, outfile_name)))) => {
info!("Job submitted: {} (UID: {})", outfile_name, uid);
submitted_jobs.push((uid, outfile_name));
}
Ok(Ok(None)) => {
debug!("Job submission returned no UID");
}
Ok(Err(e)) => {
error!("Job submission error: {}", e);
}
Err(e) => {
error!("Task join error: {}", e);
}
} else {
let status = response.status();
let text = response.text().await.unwrap_or_default();
error!("Failed to submit job ({}): {}", status, text);
Ok(None)
}
}));
}
}
// Ждём завершения ВСЕХ задач
for task in all_tasks {
match task.await {
Ok(Ok(Some((uid, outfile_name)))) => {
info!("Job submitted: {} (UID: {})", outfile_name, uid);
submitted_jobs.push((uid, outfile_name));
}
Ok(Ok(None)) => {
debug!("Job submission returned no UID");
}
Ok(Err(e)) => {
error!("Job submission error: {}", e);
}
Err(e) => {
error!("Task join error: {}", e);
}
}
}
info!(
"Successfully submitted {} jobs to Nexrender",
submitted_jobs.len()