t/audio-conv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add input file matching through configs

Browse Source 
and allow codec options for each matching
This commit is contained in:
Thomas Heck
2020-11-28 16:43:13 +01:00
parent dac57fa4d1
commit 458b7b9aa1
4 changed files with 225 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@@ -17,3 +17,4 @@ anyhow = "1"
clap = "2"
serde = { version = "1.0", features = ["derive"] }
serde_yaml = "0.8"
regex = "1"

8
example.audio-conv.yaml Normal file
View File

@@ -0,0 +1,8 @@
from: ./music
to: ./converted_test
matches:
- regex: \.flac$
to:
codec: opus
bitrate: 160
bitrate_type: vbr

115
src/config.rs
View File

@@ -1,23 +1,86 @@
use anyhow::{Context, Error, Result};
use serde::{Deserialize, Serialize};
use regex::bytes::{Regex, RegexBuilder};
use serde::Deserialize;
use std::path::{Path, PathBuf};
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct Config {
pub from: PathBuf,
pub to: PathBuf,
pub matches: Vec<TranscodeMatch>,
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Clone, Debug)]
pub struct TranscodeMatch {
pub regex: Regex,
pub to: Transcode,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "codec")]
pub enum Transcode {
#[serde(rename = "opus")]
Opus {
#[serde(default = "default_opus_bitrate")]
bitrate: u16,
#[serde(default = "default_opus_bitrate_type")]
bitrate_type: OpusBitrateType,
},
}
impl Transcode {
pub fn extention(&self) -> &'static str {
match self {
Transcode::Opus { .. } => "opus",
}
}
}
fn default_opus_bitrate() -> u16 {
160
}
fn default_opus_bitrate_type() -> OpusBitrateType {
OpusBitrateType::Vbr
}
impl Default for Transcode {
fn default() -> Self {
Transcode::Opus {
bitrate: default_opus_bitrate(),
bitrate_type: default_opus_bitrate_type(),
}
}
}
#[derive(Clone, Debug, Deserialize)]
pub enum OpusBitrateType {
#[serde(rename = "cbr")]
Cbr,
#[serde(rename = "vbr")]
Vbr,
}
#[derive(Debug, Default, Deserialize)]
struct ConfigFile {
pub from: Option<PathBuf>,
pub to: Option<PathBuf>,
from: Option<PathBuf>,
to: Option<PathBuf>,
#[serde(default)]
matches: Vec<TranscodeMatchFile>,
}
#[derive(Debug, Deserialize)]
struct TranscodeMatchFile {
regex: String,
to: Transcode,
}
pub fn config() -> Result<Config> {
use clap::Arg;
let matches = clap::App::new("audio-conv")
let arg_matches = clap::App::new("audio-conv")
.version(clap::crate_version!())
.about("Converts audio files")
.arg(
@@ -48,7 +111,7 @@ pub fn config() -> Result<Config> {
let current_dir = std::env::current_dir().context("could not get current directory")?;
let config_path = matches.value_of_os("config");
let config_path = arg_matches.value_of_os("config");
let force_load = config_path.is_some();
let config_path = config_path
.map(AsRef::<Path>::as_ref)
@@ -69,9 +132,40 @@ pub fn config() -> Result<Config> {
)));
}
let transcode_matches = config_file
.as_ref()
.map(|config_file| {
config_file
.matches
.iter()
.map(|m| {
Ok(TranscodeMatch {
regex: RegexBuilder::new(&m.regex)
.case_insensitive(true)
.build()
.context("failed compiling regex")?,
to: m.to.clone(),
})
})
.collect::<Result<Vec<_>>>()
})
.transpose()?
.filter(|matches| !matches.is_empty())
.unwrap_or_else(|| {
let default_regex = RegexBuilder::new("\\.flac$")
.case_insensitive(true)
.build()
.expect("failed compiling default match regex");
vec![TranscodeMatch {
regex: default_regex,
to: Transcode::default(),
}]
});
Ok(Config {
from: {
matches
arg_matches
.value_of_os("from")
.map(|p| current_dir.join(p))
.or_else(|| {
@@ -85,7 +179,7 @@ pub fn config() -> Result<Config> {
.canonicalize()
.context("could not canonicalize \"from\" path")?
},
to: matches
to: arg_matches
.value_of_os("to")
.map(|p| current_dir.join(p))
.or_else(|| {
@@ -98,6 +192,7 @@ pub fn config() -> Result<Config> {
.ok_or_else(|| Error::msg("\"to\" not configured"))?
.canonicalize()
.context("could not canonicalize \"to\" path")?,
matches: transcode_matches,
})
}
@@ -108,6 +203,6 @@ fn load_config_file(path: &Path) -> Result<Option<ConfigFile>> {
Err(err) => return Err(Error::new(err)),
};
let config: ConfigFile =
serde_yaml::from_reader(&mut file).context("could not read config file")?;
serde_yaml::from_reader(&mut file).context("could not parse config file")?;
Ok(Some(config))
}

134
src/main.rs
View File

@@ -1,5 +1,6 @@
mod config;
use crate::config::Config;
use anyhow::{Context, Result};
use futures::{channel::mpsc, prelude::*};
use glib::GString;
@@ -7,18 +8,19 @@ use gstreamer::Element;
use gstreamer_audio::{prelude::*, AudioEncoder};
use gstreamer_base::prelude::*;
use std::{
borrow::Cow,
ffi,
path::{Path, PathBuf},
};
fn gmake<T: IsA<Element>>(factory_name: &str) -> Result<T> {
let res = gstreamer::ElementFactory::make(factory_name, None)
.with_context(|| format!("could not make {}", factory_name))?
.with_context(|| format!("could not make gstreamer Element \"{}\"", factory_name))?
.downcast()
.ok()
.with_context(|| {
format!(
"could not cast {} into `{}`",
"could not cast gstreamer Element \"{}\" into `{}`",
factory_name,
std::any::type_name::<T>()
)
@@ -26,32 +28,56 @@ fn gmake<T: IsA<Element>>(factory_name: &str) -> Result<T> {
Ok(res)
}
fn get_path_pairs(input: PathBuf, output: PathBuf) -> impl Iterator<Item = (PathBuf, PathBuf)> {
walkdir::WalkDir::new(input.as_path())
struct ConvertionArgs {
from: PathBuf,
to: PathBuf,
transcode: config::Transcode,
}
fn get_path_pairs(config: Config) -> impl Iterator<Item = ConvertionArgs> {
walkdir::WalkDir::new(&config.from)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_type().is_file())
.filter(|e| {
e.path()
.extension()
.map(|ext| ext == "flac")
.unwrap_or(false)
})
.map(move |e| {
let mut out = output.join(e.path().strip_prefix(&input).unwrap());
out.set_extension("opus");
(e, out)
})
.filter(|(e, out)| {
let in_mtime = e.metadata().unwrap().modified().unwrap();
let out_mtime = out.metadata().and_then(|md| md.modified());
match out_mtime {
Ok(out_mtime) => out_mtime < in_mtime,
.filter_map(move |e| {
let from_bytes = path_to_bytes(e.path());
let transcode = config
.matches
.iter()
.filter(|m| m.regex.is_match(from_bytes.as_ref()))
.map(|m| m.to.clone())
.next();
let transcode = if let Some(transcode) = transcode {
transcode
} else {
return None;
};
let mut to = config.to.join(e.path().strip_prefix(&config.from).unwrap());
to.set_extension(transcode.extention());
let is_newer = {
// TODO: error handling
let from_mtime = e.metadata().unwrap().modified().unwrap();
let to_mtime = to.metadata().and_then(|md| md.modified());
match to_mtime {
Ok(to_mtime) => to_mtime < from_mtime,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => true,
Err(err) => panic!(err),
}
};
if !is_newer {
return None;
}
Some(ConvertionArgs {
from: e.path().to_path_buf(),
to,
transcode,
})
})
.map(|(e, out)| (e.into_path(), out))
}
fn main() -> Result<()> {
@@ -62,16 +88,21 @@ fn main() -> Result<()> {
// move blocking directory reading to an external thread
let pair_producer = std::thread::spawn(|| {
let produce_pairs = futures::stream::iter(get_path_pairs(config.from, config.to))
let produce_pairs = futures::stream::iter(get_path_pairs(config))
.map(Ok)
.forward(pair_tx)
.map(|res| res.context("sending path pairs failed"));
futures::executor::block_on(produce_pairs)
});
let transcoder = pair_rx.for_each_concurrent(num_cpus::get(), |(src, dest)| async move {
if let Err(err) = transcode(src.as_path(), dest.as_path()).await {
println!("err {} => {}:\n{:?}", src.display(), dest.display(), err);
let transcoder = pair_rx.for_each_concurrent(num_cpus::get(), |args| async move {
if let Err(err) = transcode(&args).await {
println!(
"err {} => {}:\n{:?}",
args.from.display(),
args.to.display(),
err
);
}
});
futures::executor::block_on(transcoder);
@@ -83,13 +114,13 @@ fn main() -> Result<()> {
Ok(())
}
async fn transcode(src: &Path, dest: &Path) -> Result<()> {
async fn transcode(args: &ConvertionArgs) -> Result<()> {
let file_src: gstreamer_base::BaseSrc = gmake("filesrc")?;
file_src.set_property("location", &path_to_gstring(src))?;
file_src.set_property("location", &path_to_gstring(&args.from))?;
// encode into a tmp file first, then rename to actuall file name, that way we're writing
// "whole" files to the intended file path, ignoring partial files in the mtime check
let tmp_dest = dest.with_extension("tmp");
let tmp_dest = args.to.with_extension("tmp");
let file_dest: gstreamer_base::BaseSink = gmake("filesink")?;
file_dest.set_property("location", &path_to_gstring(&tmp_dest))?;
file_dest.set_sync(false);
@@ -99,9 +130,24 @@ async fn transcode(src: &Path, dest: &Path) -> Result<()> {
resample.set_property("quality", &7)?;
let encoder: AudioEncoder = gmake("opusenc")?;
encoder.set_property("bitrate", &160_000)?;
// 0 = cbr; 1 = vbr
encoder.set_property_from_str("bitrate-type", "1");
let config::Transcode::Opus {
bitrate,
bitrate_type,
} = &args.transcode;
encoder.set_property(
"bitrate",
&i32::from(*bitrate)
.checked_mul(1_000)
.context("bitrate overflowed")?,
)?;
encoder.set_property_from_str(
"bitrate-type",
match bitrate_type {
config::OpusBitrateType::Vbr => "1",
config::OpusBitrateType::Cbr => "0",
},
);
let elems: &[&Element] = &[
file_src.upcast_ref(),
@@ -123,8 +169,9 @@ async fn transcode(src: &Path, dest: &Path) -> Result<()> {
let bus = pipeline.get_bus().context("pipe get bus")?;
std::fs::create_dir_all(
dest.parent()
.with_context(|| format!("could not get parent dir for {}", dest.display()))?,
args.to
.parent()
.with_context(|| format!("could not get parent dir for {}", args.to.display()))?,
)?;
rm_file_on_err(&tmp_dest, async {
@@ -158,7 +205,7 @@ async fn transcode(src: &Path, dest: &Path) -> Result<()> {
.set_state(gstreamer::State::Null)
.context("Unable to set the pipeline to the `Null` state")?;
std::fs::rename(&tmp_dest, dest)?;
std::fs::rename(&tmp_dest, &args.to)?;
Ok(())
})
@@ -181,19 +228,17 @@ where
}
}
fn path_to_gstring(path: &Path) -> GString {
let buf = {
let mut buf = Vec::<u8>::new();
fn path_to_bytes(path: &Path) -> Cow<'_, [u8]> {
// https://stackoverflow.com/a/59224987/5572146
#[cfg(unix)]
{
use std::os::unix::ffi::OsStrExt;
buf.extend(path.as_os_str().as_bytes());
Cow::Borrowed(path.as_os_str().as_bytes())
}
#[cfg(windows)]
{
let mut buf = Vec::<u8>::new();
// NOT TESTED
// FIXME: test and post answer to https://stackoverflow.com/questions/38948669
use std::os::windows::ffi::OsStrExt;
@@ -203,10 +248,13 @@ fn path_to_gstring(path: &Path) -> GString {
.map(|char| char.to_ne_bytes())
.flatten(),
);
Cow::Owned(buf)
}
}
buf
};
ffi::CString::new(buf).unwrap().into()
fn path_to_gstring(path: &Path) -> GString {
let buf = path_to_bytes(path);
ffi::CString::new(buf)
.expect("Path contained null byte")
.into()
}