|
|
|
@ -28,9 +28,7 @@ fn override_stars(level: u32, text: &str) -> Option<u32> {
|
|
|
|
|
// This is zero because a lot of the resources are non-github/non-cargo links and overriding for all would be annoying
|
|
|
|
|
// These should be evaluated with more primitive means
|
|
|
|
|
Some(0)
|
|
|
|
|
} else if level == 3 && text.contains("Games") {
|
|
|
|
|
Some(40)
|
|
|
|
|
} else if level == 3 && text.contains("Emulators") {
|
|
|
|
|
} else if level == 3 && (text.contains("Games") || text.contains("Emulators")) {
|
|
|
|
|
Some(40)
|
|
|
|
|
} else {
|
|
|
|
|
None // i.e. use defaults
|
|
|
|
@ -133,9 +131,9 @@ impl MaxHandles {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async fn get<'a>(&'a self) -> Handle<'a> {
|
|
|
|
|
async fn get(&self) -> Handle {
|
|
|
|
|
let permit = self.remaining.acquire().await.unwrap();
|
|
|
|
|
return Handle { _permit: permit };
|
|
|
|
|
Handle { _permit: permit }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -162,7 +160,7 @@ fn get_url(url: String) -> BoxFuture<'static, (String, Result<(), CheckerError>)
|
|
|
|
|
debug!("Need handle for {}", url);
|
|
|
|
|
async move {
|
|
|
|
|
let _handle = HANDLES.get().await;
|
|
|
|
|
return get_url_core(url).await;
|
|
|
|
|
get_url_core(url).await
|
|
|
|
|
}
|
|
|
|
|
.boxed()
|
|
|
|
|
}
|
|
|
|
@ -184,7 +182,7 @@ struct GithubStars {
|
|
|
|
|
async fn get_stars(github_url: &str) -> Option<u32> {
|
|
|
|
|
warn!("Downloading Github stars for {}", github_url);
|
|
|
|
|
let rewritten = GITHUB_REPO_REGEX
|
|
|
|
|
.replace_all(&github_url, "https://api.github.com/repos/$org/$repo")
|
|
|
|
|
.replace_all(github_url, "https://api.github.com/repos/$org/$repo")
|
|
|
|
|
.to_string();
|
|
|
|
|
let mut req = CLIENT.get(&rewritten);
|
|
|
|
|
if let Ok(username) = env::var("USERNAME_FOR_GITHUB") {
|
|
|
|
@ -198,7 +196,7 @@ async fn get_stars(github_url: &str) -> Option<u32> {
|
|
|
|
|
match resp {
|
|
|
|
|
Err(err) => {
|
|
|
|
|
warn!("Error while getting {}: {}", github_url, err);
|
|
|
|
|
return None;
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
Ok(ok) => {
|
|
|
|
|
let raw = ok.text().await.unwrap();
|
|
|
|
@ -212,7 +210,7 @@ async fn get_stars(github_url: &str) -> Option<u32> {
|
|
|
|
|
warn!("{} is archived, so ignoring stars", github_url);
|
|
|
|
|
return Some(0);
|
|
|
|
|
}
|
|
|
|
|
return Some(data.stargazers_count);
|
|
|
|
|
Some(data.stargazers_count)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -231,7 +229,7 @@ struct Crate {
|
|
|
|
|
async fn get_downloads(github_url: &str) -> Option<u64> {
|
|
|
|
|
warn!("Downloading Crates downloads for {}", github_url);
|
|
|
|
|
let rewritten = CRATE_REGEX
|
|
|
|
|
.replace_all(&github_url, "https://crates.io/api/v1/crates/$crate")
|
|
|
|
|
.replace_all(github_url, "https://crates.io/api/v1/crates/$crate")
|
|
|
|
|
.to_string();
|
|
|
|
|
let req = CLIENT.get(&rewritten);
|
|
|
|
|
|
|
|
|
@ -239,11 +237,11 @@ async fn get_downloads(github_url: &str) -> Option<u64> {
|
|
|
|
|
match resp {
|
|
|
|
|
Err(err) => {
|
|
|
|
|
warn!("Error while getting {}: {}", github_url, err);
|
|
|
|
|
return None;
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
Ok(ok) => {
|
|
|
|
|
let data = ok.json::<Crate>().await.unwrap();
|
|
|
|
|
return Some(data.info.downloads);
|
|
|
|
|
Some(data.info.downloads)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -356,7 +354,7 @@ fn get_url_core(url: String) -> BoxFuture<'static, (String, Result<(), CheckerEr
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
let query = matches.get(1).map(|x| x.as_str()).unwrap_or("");
|
|
|
|
|
if !query.starts_with("?") || query.find("branch=").is_none() {
|
|
|
|
|
if !query.starts_with('?') || !query.contains("branch=") {
|
|
|
|
|
res = Err(CheckerError::TravisBuildNoBranch);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
@ -402,7 +400,7 @@ async fn main() -> Result<(), Error> {
|
|
|
|
|
let mut results: Results = fs::read_to_string("results/results.yaml")
|
|
|
|
|
.map_err(|e| format_err!("{}", e))
|
|
|
|
|
.and_then(|x| serde_yaml::from_str(&x).map_err(|e| format_err!("{}", e)))
|
|
|
|
|
.unwrap_or(Results::new());
|
|
|
|
|
.unwrap_or_default();
|
|
|
|
|
|
|
|
|
|
let mut popularity_data: PopularityData = fs::read_to_string("results/popularity.yaml")
|
|
|
|
|
.map_err(|e| format_err!("{}", e))
|
|
|
|
@ -460,7 +458,7 @@ async fn main() -> Result<(), Error> {
|
|
|
|
|
Event::Start(tag) => {
|
|
|
|
|
match tag {
|
|
|
|
|
Tag::Link(_link_type, url, _title) | Tag::Image(_link_type, url, _title) => {
|
|
|
|
|
if !url.starts_with("#") {
|
|
|
|
|
if !url.starts_with('#') {
|
|
|
|
|
let new_url = url.to_string();
|
|
|
|
|
if POPULARITY_OVERRIDES.contains(&new_url) {
|
|
|
|
|
github_stars = Some(MINIMUM_GITHUB_STARS);
|
|
|
|
@ -520,14 +518,14 @@ async fn main() -> Result<(), Error> {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
Tag::List(_) => {
|
|
|
|
|
if in_list_item && list_item.len() > 0 {
|
|
|
|
|
if in_list_item && !list_item.is_empty() {
|
|
|
|
|
list_items.last_mut().unwrap().data.push(list_item.clone());
|
|
|
|
|
in_list_item = false;
|
|
|
|
|
}
|
|
|
|
|
list_items.push(ListInfo { data: Vec::new() });
|
|
|
|
|
}
|
|
|
|
|
Tag::Item => {
|
|
|
|
|
if in_list_item && list_item.len() > 0 {
|
|
|
|
|
if in_list_item && !list_item.is_empty() {
|
|
|
|
|
list_items.last_mut().unwrap().data.push(list_item.clone());
|
|
|
|
|
}
|
|
|
|
|
in_list_item = true;
|
|
|
|
@ -567,19 +565,18 @@ async fn main() -> Result<(), Error> {
|
|
|
|
|
Event::End(tag) => {
|
|
|
|
|
match tag {
|
|
|
|
|
Tag::Item => {
|
|
|
|
|
if list_item.len() > 0 {
|
|
|
|
|
if link_count > 0 {
|
|
|
|
|
if github_stars.unwrap_or(0) < required_stars
|
|
|
|
|
&& cargo_downloads.unwrap_or(0) < MINIMUM_CARGO_DOWNLOADS
|
|
|
|
|
{
|
|
|
|
|
if github_stars.is_none() {
|
|
|
|
|
warn!("No valid github link");
|
|
|
|
|
}
|
|
|
|
|
if cargo_downloads.is_none() {
|
|
|
|
|
warn!("No valid crates link");
|
|
|
|
|
}
|
|
|
|
|
return Err(format_err!("Not high enough metrics ({:?} stars < {}, and {:?} cargo downloads < {}): {}", github_stars, required_stars, cargo_downloads, MINIMUM_CARGO_DOWNLOADS, list_item));
|
|
|
|
|
if !list_item.is_empty() {
|
|
|
|
|
if link_count > 0
|
|
|
|
|
&& github_stars.unwrap_or(0) < required_stars
|
|
|
|
|
&& cargo_downloads.unwrap_or(0) < MINIMUM_CARGO_DOWNLOADS
|
|
|
|
|
{
|
|
|
|
|
if github_stars.is_none() {
|
|
|
|
|
warn!("No valid github link");
|
|
|
|
|
}
|
|
|
|
|
if cargo_downloads.is_none() {
|
|
|
|
|
warn!("No valid crates link");
|
|
|
|
|
}
|
|
|
|
|
return Err(format_err!("Not high enough metrics ({:?} stars < {}, and {:?} cargo downloads < {}): {}", github_stars, required_stars, cargo_downloads, MINIMUM_CARGO_DOWNLOADS, list_item));
|
|
|
|
|
}
|
|
|
|
|
list_items.last_mut().unwrap().data.push(list_item.clone());
|
|
|
|
|
list_item = String::new();
|
|
|
|
@ -588,18 +585,18 @@ async fn main() -> Result<(), Error> {
|
|
|
|
|
}
|
|
|
|
|
Tag::List(_) => {
|
|
|
|
|
let list_info = list_items.pop().unwrap();
|
|
|
|
|
if list_info.data.iter().find(|s| *s == "License").is_some()
|
|
|
|
|
&& list_info.data.iter().find(|s| *s == "Resources").is_some()
|
|
|
|
|
if list_info.data.iter().any(|s| *s == "License")
|
|
|
|
|
&& list_info.data.iter().any(|s| *s == "Resources")
|
|
|
|
|
{
|
|
|
|
|
// Ignore wrong ordering in top-level list
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
let mut sorted_recent_list = list_info.data.to_vec();
|
|
|
|
|
sorted_recent_list.sort_by(|a, b| a.to_lowercase().cmp(&b.to_lowercase()));
|
|
|
|
|
sorted_recent_list.sort_by_key(|a| a.to_lowercase());
|
|
|
|
|
let joined_recent = list_info.data.join("\n");
|
|
|
|
|
let joined_sorted = sorted_recent_list.join("\n");
|
|
|
|
|
let patch = create_patch(&joined_recent, &joined_sorted);
|
|
|
|
|
if patch.hunks().len() > 0 {
|
|
|
|
|
if !patch.hunks().is_empty() {
|
|
|
|
|
println!("{}", patch);
|
|
|
|
|
return Err(format_err!("Sorting error"));
|
|
|
|
|
}
|
|
|
|
@ -625,30 +622,14 @@ async fn main() -> Result<(), Error> {
|
|
|
|
|
)?;
|
|
|
|
|
|
|
|
|
|
to_check.sort_by(|a, b| {
|
|
|
|
|
let get_time = |k| {
|
|
|
|
|
let res = results.get(k);
|
|
|
|
|
if let Some(link) = res {
|
|
|
|
|
if let Some(last_working) = link.last_working {
|
|
|
|
|
Some(last_working)
|
|
|
|
|
} else {
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
let get_time = |k| results.get(k).map(|link| link.last_working);
|
|
|
|
|
let res_a = get_time(a);
|
|
|
|
|
let res_b = get_time(b);
|
|
|
|
|
if res_a.is_none() {
|
|
|
|
|
if res_b.is_none() {
|
|
|
|
|
return a.cmp(b);
|
|
|
|
|
} else {
|
|
|
|
|
Ordering::Less
|
|
|
|
|
}
|
|
|
|
|
} else if res_b.is_none() {
|
|
|
|
|
Ordering::Greater
|
|
|
|
|
} else {
|
|
|
|
|
res_a.unwrap().cmp(&res_b.unwrap())
|
|
|
|
|
match (res_a, res_b) {
|
|
|
|
|
(Some(a), Some(b)) => a.cmp(&b),
|
|
|
|
|
(Some(_), None) => Ordering::Less,
|
|
|
|
|
(None, Some(_)) => Ordering::Greater,
|
|
|
|
|
(None, None) => a.cmp(b),
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
@ -665,7 +646,7 @@ async fn main() -> Result<(), Error> {
|
|
|
|
|
|
|
|
|
|
let mut not_written = 0;
|
|
|
|
|
let mut last_written = Local::now();
|
|
|
|
|
while url_checks.len() > 0 {
|
|
|
|
|
while !url_checks.is_empty() {
|
|
|
|
|
debug!("Waiting for {}", url_checks.len());
|
|
|
|
|
let ((url, res), _index, remaining) = select_all(url_checks).await;
|
|
|
|
|
url_checks = remaining;
|
|
|
|
@ -715,7 +696,7 @@ async fn main() -> Result<(), Error> {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
fs::write("results/results.yaml", serde_yaml::to_string(&results)?)?;
|
|
|
|
|
println!("");
|
|
|
|
|
println!();
|
|
|
|
|
let mut failed: u32 = 0;
|
|
|
|
|
|
|
|
|
|
for (url, link) in results.iter() {
|
|
|
|
|