Wyam.Common.Tracing.Trace.Information($"Getting .NET Foundation readme");

string foundationReadme;

using(System.Net.WebClient webClient = new System.Net.WebClient())

{

foundationReadme = await webClient.DownloadStringTaskAsync("https://raw.githubusercontent.com/dotnet/home/master/README.md");

}

// Even though it's much slower, we have to query the GitHub API sequentially or else we'll trigger the abuse detection mechanisms:

// "Make requests for a single user or client ID serially. Do not make requests for a single user or client ID concurrently."

// https://developer.github.com/v3/guides/best-practices-for-integrators/#dealing-with-abuse-rate-limits

List<IDocument> outputs = new List<IDocument>();

foreach(IDocument input in inputs)

{

IDocument output = input;

// Extract the GitHub owner and name

Uri source;

if(Uri.TryCreate(input.String("Source"), UriKind.Absolute, out source)

&& source.Host.EndsWith("github.com", StringComparison.OrdinalIgnoreCase))

{

string owner = source.Segments[1].Trim('/');

string name = source.Segments[2].Trim('/');

// Connect to GitHub and get the repository

Wyam.Common.Tracing.Trace.Information($"Getting GitHub data for {owner}/{name}");

Repository repository = await github.Repository.Get(owner, name);

// Get the metadata

MetadataItems metadata = new MetadataItems

{

{ "StargazersCount", repository.StargazersCount },

{ "ForksCount", repository.ForksCount },

{ "OpenIssuesCount", repository.OpenIssuesCount },

{ "PushedAt", repository.PushedAt },

{ "CreatedAt", repository.CreatedAt }

};

if(!input.ContainsKey("Title"))

{

metadata.Add("Title", repository.Name);

}

if(!input.ContainsKey("Description"))

{

metadata.Add("Description", repository.Description);

}

if(!input.ContainsKey("Website"))

{

metadata.Add("Website", repository.Homepage);

}

if(!input.ContainsKey("Microsoft") && microsoftOwners.Contains(owner, StringComparer.OrdinalIgnoreCase))

{

metadata.Add("Microsoft", true);

}

if(!input.ContainsKey("Foundation") && foundationReadme.IndexOf($"github.com/{owner}/{name}", StringComparison.OrdinalIgnoreCase) >= 0)

{

metadata.Add("Foundation", true);

}

// Get the readme

Wyam.Common.Tracing.Trace.Information($"Getting GitHub readme for {owner}/{name}");

try

{

string readme = await github.Repository.Content.GetReadmeHtml(owner, name);

if(!string.IsNullOrEmpty(readme))

{

metadata.Add("Readme", readme);

}

}

catch(Exception ex)

{

Wyam.Common.Tracing.Trace.Information($"Exception getting GitHub readme for {owner}/{name}: {ex.Message}");

}

// Report on out API rate limit

MiscellaneousRateLimit rateLimit = await github.Miscellaneous.GetRateLimits();

Wyam.Common.Tracing.Trace.Information($"GitHub rate limit: {rateLimit.Resources.Core.Remaining} remaining");

// Return a document with GitHub metadata

output = context.GetDocument(input, metadata);

}

outputs.Add(output);

}

return outputs;

}

public class VersionData

{

public string Version { get; }

public string Downloads { get; }

public DateTime LastUpdated { get; }

public VersionData(AngleSharp.Dom.IElement element)

{

Version = element.Children[1].TextContent.Replace("(current)", string.Empty).Trim();

Downloads = element.Children[2].TextContent.Trim();

LastUpdated = DateTime.Parse(element.QuerySelector("*[data-datetime]").GetAttribute("data-datetime"));

}

}

public async Task<IEnumerable<IDocument>> GetProjectNuGetData(IReadOnlyList<IDocument> inputs, IExecutionContext context)

{

AngleSharp.IConfiguration config = AngleSharp.Configuration.Default.WithDefaultLoader();

List<IDocument> outputs = new List<IDocument>();

foreach(IDocument input in inputs)

{

IDocument output = input;

string package = input.String("NuGet");

if(!string.IsNullOrEmpty(package))

{

Wyam.Common.Tracing.Trace.Information($"Getting NuGet data for {package}");

try

{

IBrowsingContext browsingContext = BrowsingContext.New(config);

AngleSharp.Dom.IDocument document = await browsingContext.OpenAsync($"https://www.nuget.org/packages/{package}");

if(document.StatusCode != System.Net.HttpStatusCode.OK)

{

Wyam.Common.Tracing.Trace.Warning($"Bad status code for {package}: {document.StatusCode}");

}

else if(document == null)

{

Wyam.Common.Tracing.Trace.Warning($"Could not get document for {package}");

}

else

{

// Get statistics

AngleSharp.Dom.IElement statistics = document

.QuerySelectorAll(".package-details-info h2")

.First(x => x.TextContent == "Statistics")

.NextElementSibling;

string totalDownloads = statistics.Children

.First(x => x.TextContent.Contains("total downloads"))

.TextContent.Trim().Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)[0];

string latestVersionDownloads = statistics.Children

.First(x => x.TextContent.Contains("latest version"))

.TextContent.Trim().Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)[0];

string perDayDownloads = statistics.Children

.First(x => x.TextContent.Contains("per day"))

.TextContent.Trim().Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)[0];

// Get versions

List<VersionData> versions = document

.QuerySelectorAll("#version-history table tbody tr")

.Select(x => new VersionData(x))

.ToList();

// Return a document with NuGet data

output = context.GetDocument(input, new MetadataItems

{

{ "TotalDownloads", totalDownloads },

{ "LatestVersionDownloads", latestVersionDownloads },

{ "PerDayDownloads", perDayDownloads },

{ "LatestVersion", versions.FirstOrDefault()?.Version ?? string.Empty },

{ "LatestVersionLastUpdated", versions.FirstOrDefault()?.LastUpdated },

{ "Versions", versions }

});

}

}

catch(Exception ex)

{

Wyam.Common.Tracing.Trace.Warning($"Error getting NuGet data for {package}: {ex.Message}");

}

}

outputs.Add(output);

}

return outputs;

}

public class FeedItemData

{

public string Title { get; }

public string Link { get; }

public string Description { get; }

public DateTimeOffset Published { get; }

public bool Recent { get; }

public IDictionary<string, string> Links { get; }

public string Author { get; }

public FeedItemData(ISyndicationItem item, DateTimeOffset recent)

{

Title = item.Title;

Link = item.Links.FirstOrDefault(x => x.RelationshipType == RssLinkTypes.Alternate)?.Uri.ToString() ?? item.Id;

Published = item.Published != default(DateTimeOffset) ? item.Published : item.LastUpdated;

Recent = Published > recent;

Description = item.Description;

Links = item.Links

.Where(x => !string.IsNullOrEmpty(x.MediaType))

.GroupBy(x => x.MediaType)

.Select(x => x.First())

.ToDictionary(x => x.MediaType, x => x.Uri.ToString());

ISyndicationPerson person = item.Contributors.FirstOrDefault(x => x.RelationshipType == "author");

if(person != null)

{

Author = person.Name ?? person.Email;

}

AtomEntry atom = item as AtomEntry;

if(atom != null && !string.IsNullOrEmpty(atom.Summary))

{

Description = atom.Summary;

}

}

}

public async Task<IEnumerable<IDocument>> GetFeedData(IReadOnlyList<IDocument> inputs, IExecutionContext context)

{

DateTimeOffset recent = new DateTimeOffset(DateTime.Today.AddHours(context.Bool("Preview") ? -168 : -48));

return await Task.WhenAll(

inputs.Select(async input =>

{

string feed = input.String("Feed");

if(!string.IsNullOrEmpty(feed))

{

try

{

// Download the feed

Wyam.Common.Tracing.Trace.Information($"Getting feed for {feed}");

string website = null;

string title = null;

string author = null;

string description = null;

string image = null;

List<ISyndicationItem> items = new List<ISyndicationItem>();

using (WebClient webClient = new WebClient())

{

using (Stream stream = await webClient.OpenReadTaskAsync(feed))

{

using (StreamReader streamReader = new XmlStreamReader(stream))

{

using(XmlReader xmlReader = XmlReader.Create(streamReader, new XmlReaderSettings { Async = true, DtdProcessing = DtdProcessing.Ignore }))

{

xmlReader.MoveToContent();

bool atom = xmlReader.Name == "feed";

Wyam.Common.Tracing.Trace.Information($"Reading {feed} as " + (atom ? "Atom" : "RSS"));

XmlFeedReader feedReader = atom

? (XmlFeedReader)new AtomFeedReader(xmlReader, new DiscoverAtomParser())

: new RssFeedReader(xmlReader);

while(await feedReader.Read())

{

try

{

switch(feedReader.ElementType)

{

case SyndicationElementType.Person:

ISyndicationPerson person = await feedReader.ReadPerson();

if(person.RelationshipType == "author")

{

author = person.Name ?? person.Email;

}

break;

case SyndicationElementType.Image:

ISyndicationImage img = await feedReader.ReadImage();

image = img.Url.ToString();

break;

case SyndicationElementType.Link:

ISyndicationLink link = await feedReader.ReadLink();

website = link.Uri.ToString();

break;

case SyndicationElementType.Item:

ISyndicationItem item = await feedReader.ReadItem();

items.Add(item);

break;

case SyndicationElementType.None:

break;

default:

ISyndicationContent content = await feedReader.ReadContent();

if(string.Equals(content.Name, "title", StringComparison.OrdinalIgnoreCase))

{

title = content.Value;

}

else if(string.Equals(content.Name, "description", StringComparison.OrdinalIgnoreCase)

|| string.Equals(content.Name, "subtitle", StringComparison.OrdinalIgnoreCase))

{

description = content.Value;

}

break;

}

}

catch(Exception ex)

{

Wyam.Common.Tracing.Trace.Warning($"Exception while processing {feedReader.ElementType}: {ex.Message}");

}

}

}

}

}

}

// Get a new document with feed metadata

MetadataItems metadata = new MetadataItems();

if(items.Count > 0)

{

FeedItemData[] feedItems = items

.Select(x => new FeedItemData(x, recent))

.OrderByDescending(x => x.Published)

.Take(50) // Only take the 50 most recent items

.ToArray();

metadata.Add("FeedItems", feedItems);

metadata.Add("LastPublished", feedItems.First().Published);

metadata.Add("NewestFeedItem", feedItems[0]);

};

if(!input.ContainsKey("Website") && !string.IsNullOrEmpty(website))

{

metadata.Add("Website", website);

}

if(!input.ContainsKey("Title") && !string.IsNullOrEmpty(title))

{

metadata.Add("Title", title);

}

if(!input.ContainsKey("Author") && !string.IsNullOrEmpty(author))

{

metadata.Add("Author", author);

}

if(!input.ContainsKey("Description") && !string.IsNullOrEmpty(description))

{

metadata.Add("Description", description);

}

if(!input.ContainsKey("Image") && !string.IsNullOrEmpty(image))

{

metadata.Add("Image", image);

}

return context.GetDocument(input, metadata);

}

catch(Exception ex)

{

Wyam.Common.Tracing.Trace.Warning($"Error getting feed for {feed}: {ex.Message}");

}

}

return null;

}));

}

// See https://github.com/dotnet/SyndicationFeedReaderWriter/issues/31

public class DiscoverAtomParser : AtomParser

{

public override IAtomEntry CreateEntry(ISyndicationContent content)

{

// Remove author and contributor entries if they don't contain an email or name

ICollection<ISyndicationContent> children = (ICollection<ISyndicationContent>)content.Fields;

ISyndicationContent author = children.FirstOrDefault(x => x.Name == AtomContributorTypes.Author);

if(author != null

&& author.Fields.FirstOrDefault(x => x.Name == "name")?.Value == null

&& author.Fields.FirstOrDefault(x => x.Name == "email")?.Value == null)

{

children.Remove(author);

}

ISyndicationContent contributor = children.FirstOrDefault(x => x.Name == AtomContributorTypes.Contributor);

if (contributor != null

&& contributor.Fields.FirstOrDefault(x => x.Name == "name")?.Value == null

&& contributor.Fields.FirstOrDefault(x => x.Name == "email")?.Value == null)

{

children.Remove(contributor);

}

return base.CreateEntry(content);

}

}

public class XmlStreamReader : StreamReader

{

public XmlStreamReader(Stream stream) : base(stream)

{

}

public override int Peek()

{

int ch = 0;

while (ch != -1)

{

ch = base.Peek();

if (XmlConvert.IsXmlChar((char)ch))

{

return ch;

}

base.Read();

}

return ch;

}

public override int Read()

{

int ch = 0;

while (ch != -1)

{

ch = base.Read();

if (XmlConvert.IsXmlChar((char)ch))

{

return ch;

}

}

return ch;

}

public override int Read(char[] buffer, int index, int count)

{

int i;

for(i = index ; i < count ; i++)

{

int c = Read();

if(c == -1)

{

break;

}

buffer[i] = (char)c;

}

return i - index;