Support password-protected zip files

Cargo.toml

@@ -15,13 +15,14 @@ path = "src/main.rs"
 
 [dependencies]
 anyhow = "1.0"
-clap = { version = "4.5", features = ["derive"] }
+clap = { version = "4.6", features = ["derive"] }
 crc32fast = "1.5"
 nohash-hasher = "0.2.0"
 quick-xml = "0.39.2"
 smallvec = "1.15"
-sha1_smol = { version = "1.0", features = ["std"] }
 flate2 = "1"
 lzma-rs = "0.3"
-zip = { version = "8", default-features = false, features = ["deflate"] }
+zip = { version = "8", default-features = false, features = ["aes-crypto", "deflate-flate2"] }
+sha1 = "0.10"
 zstd = "0.13"
+

src/gex.rs

@@ -104,7 +104,11 @@ class GeneratedTask(BaseTask):
  ));
  out.push_str(" contents = f.read()\n");
  }
- CandidateSource::Zip { archive, member } => {
+ CandidateSource::Zip {
+ archive,
+ member,
+ password,
+ } => {
  let archive_name = archive
  .file_name()
  .map(|s| s.to_string_lossy())
@@ -114,7 +118,20 @@ class GeneratedTask(BaseTask):
  out.push_str(&format!(
  " with zipfile.ZipFile(os.path.join(in_dir, {py_archive})) as z:\n"
  ));
- out.push_str(&format!(" with z.open({py_member}) as f:\n"));
+ if let Some(pw) = password {
+ let py_pw = py_str(pw);
+ out.push_str(
+ " # NOTE: zipfile only supports ZipCrypto; AES-encrypted\n",
+ );
+ out.push_str(
+ " # archives need pyzipper or another AES-capable library.\n",
+ );
+ out.push_str(&format!(
+ " with z.open({py_member}, pwd={py_pw}.encode()) as f:\n"
+ ));
+ } else {
+ out.push_str(&format!(" with z.open({py_member}) as f:\n"));
+ }
  out.push_str(" contents = f.read()\n");
  }
  CandidateSource::Kpka { archive, index } => {

src/lib.rs

@@ -32,7 +32,7 @@ pub fn apply_found<F>(
 where
  F: FnMut(&RomInfo, &[u8]) -> anyhow::Result<()>,
 {
- use sha1_smol::Sha1;
+ use sha1::{Digest, Sha1};
  use types::MatchedData;
 
  let MatchedData::Spec(ref spec) = found.data;
@@ -53,12 +53,12 @@ where
  let mut hasher = Sha1::new();
  hasher.update(roms[rid].header.as_ref().unwrap());
  hasher.update(&bytes_owned);
- hasher.digest().to_string()
+format!("{:x}", hasher.finalize())
  })
  .as_str()
  } else {
  if sha1_cache.is_none() {
- sha1_cache = Some(Sha1::from(&bytes_owned[..]).digest().to_string());
+ sha1_cache = Some(format!("{:x}", Sha1::digest(&bytes_owned[..])));
  }
  sha1_cache.as_ref().unwrap().as_str()
  };

src/main.rs

@@ -1,5 +1,6 @@
 #![deny(warnings)]
 
+use anyhow::Context;
 use chisel::utils;
 use clap::Parser;
 
@@ -61,14 +62,36 @@ struct Opt {
  #[arg(long)]
  no_expand: bool,
 
+ /// Passwords to try on encrypted zip members (may be repeated)
+ #[arg(long)]
+ password: Vec<String>,
+
+ /// Read passwords from a file, one per line
+ #[arg(long)]
+ password_file: Option<std::path::PathBuf>,
+
  /// Verbose logging (-v)
  #[arg(short, long)]
  verbose: bool,
 }
 
-fn run_extract(opt: &Opt, spec: chisel::types::ExtractionSpec) -> anyhow::Result<()> {
+fn load_passwords(opt: &Opt) -> anyhow::Result<Vec<String>> {
+ let mut passwords = opt.password.clone();
+ if let Some(ref path) = opt.password_file {
+ let content = std::fs::read_to_string(path)
+ .with_context(|| format!("Reading password file {}", path.display()))?;
+ passwords.extend(content.lines().filter(|l| !l.is_empty()).map(String::from));
+ }
+ Ok(passwords)
+}
+
+fn run_extract(
+ opt: &Opt,
+ passwords: &[String],
+ spec: chisel::types::ExtractionSpec,
+) -> anyhow::Result<()> {
  std::fs::create_dir_all(&opt.output_dir)?;
- let cands = utils::load_candidates_from_paths(&opt.input_files, opt.verbose)?;
+ let cands = utils::load_candidates_from_paths(&opt.input_files, passwords, opt.verbose)?;
  for cand in &cands {
  let mut s = spec.clone();
  if s.size == 0 {
@@ -101,16 +124,18 @@ fn main() -> anyhow::Result<()> {
  anyhow::bail!("--dat and --spec are mutually exclusive");
  }
 
+ let passwords = load_passwords(&opt)?;
+
  if let Some(spec) = opt.spec.clone() {
- return run_extract(&opt, spec);
+ return run_extract(&opt, &passwords, spec);
  }
 
  let dat = opt
  .dat
  .as_ref()
  .ok_or_else(|| anyhow::anyhow!("either --dat or --spec is required"))?;
  let mut roms = load_rom_list(dat, opt.game.as_deref())?;
- let mut cands = utils::load_candidates_from_paths(&opt.input_files, opt.verbose)?;
+ let mut cands = utils::load_candidates_from_paths(&opt.input_files, &passwords, opt.verbose)?;
 
  if opt.gex.is_none() {
  std::fs::create_dir_all(&opt.output_dir)?;

src/pipeline.rs

@@ -41,6 +41,7 @@ fn cost_ratio(work: u64, value: u64) -> u64 {
  ((work as f64) / (value.max(1) as f64)).ceil() as u64
 }
 
+#[allow(clippy::too_many_arguments)]
 pub fn run_pipeline(
  roms: &mut [RomInfo],
  cands: &mut [Candidate],

src/test_support.rs

@@ -1,6 +1,6 @@
 use crate::types::{Candidate, CandidateSource, Found, Heuristic, MatchRecord, Pending, RomInfo};
 use crc32fast;
-use sha1_smol::Sha1;
+use sha1::{Digest, Sha1};
 use std::collections::HashMap;
 
 pub fn make_rom(name: &str, data: &[u8]) -> RomInfo {
@@ -9,7 +9,7 @@ pub fn make_rom(name: &str, data: &[u8]) -> RomInfo {
  game: String::new(),
  size: data.len(),
  crc32: crc32fast::hash(data),
- sha1: Some(Sha1::from(data).digest().to_string()),
+ sha1: Some(format!("{:x}", Sha1::digest(data))),
  matched: false,
  unverified: false,
  region: None,
@@ -25,7 +25,7 @@ pub fn make_rom_with_header(name: &str, header: &[u8], content: &[u8]) -> RomInf
  full.extend_from_slice(header);
  full.extend_from_slice(content);
  let full_crc = crc32fast::hash(&full);
- let full_sha1 = Sha1::from(&full).digest().to_string();
+ let full_sha1 = format!("{:x}", Sha1::digest(&full));
  let content_crc = crate::utils::derive_content_crc(full_crc, header, content.len());
  RomInfo {
  name: name.to_string(),

src/types.rs

@@ -235,7 +235,11 @@ pub enum CandidateSource {
  /// Single file decompressed from a gzip stream.
  Gzip { archive: PathBuf },
  /// One member extracted from a zip archive.
- Zip { archive: PathBuf, member: String },
+ Zip {
+ archive: PathBuf,
+ member: String,
+ password: Option<String>,
+ },
  /// Decompressed from an LZMA/XZ block found at `offset` inside `parent`.
  Lzma { parent: PathBuf, offset: usize },
  /// One entry extracted from a KPKA/PAK archive (index is entry ordinal, 0-based).
@@ -268,7 +272,9 @@ impl std::fmt::Display for Candidate {
  .unwrap_or("???".into());
  write!(f, "[gzip in {}]", a)?;
  }
- CandidateSource::Zip { archive, member } => {
+ CandidateSource::Zip {
+ archive, member, ..
+ } => {
  let a = archive
  .file_name()
  .map(|s| s.to_string_lossy())