diff --git a/src/.dockerignore b/src/.dockerignore new file mode 100644 index 0000000..fe1152b --- /dev/null +++ b/src/.dockerignore @@ -0,0 +1,30 @@ +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/azds.yaml +**/bin +**/charts +**/docker-compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +LICENSE +README.md +!**/.gitignore +!.git/HEAD +!.git/config +!.git/packed-refs +!.git/refs/heads/** \ No newline at end of file diff --git a/src/Backend/Controllers/JobSearchController.cs b/src/Backend/Controllers/JobSearchController.cs index 47e329d..2ced804 100644 --- a/src/Backend/Controllers/JobSearchController.cs +++ b/src/Backend/Controllers/JobSearchController.cs @@ -2,102 +2,57 @@ namespace Backend.Controllers { using Backend.Operations; using Microsoft.AspNetCore.Mvc; - using Common.Models; using Common.Models.Public; using Common.Repositories; using System.Threading.Tasks; + using Common.Managers; + using Common.Engines; + using Common.Queries; + using Common.DatabaseModels; [ApiController] [Route("api")] public class JobSearchController : ControllerBase { - private readonly GSEngine gsEngine; - private readonly AIEngine aiEngine; - private readonly JobScrapperManager jobscrapperManager; - private readonly JobsRepository jobsContainer; + private readonly JobsRepository jobsRepository; private readonly ILogger logger; - public JobSearchController(GSEngine gsEngine, AIEngine aiEngine, JobsRepository jobsContainer, JobScrapperManager jobscrapperManager, ILogger logger) + public JobSearchController(JobsRepository jobsRepository, ILogger logger) { - this.gsEngine = gsEngine; - this.aiEngine = aiEngine; this.logger = logger; - this.jobscrapperManager = jobscrapperManager; - this.jobsContainer = jobsContainer; + this.jobsRepository = jobsRepository; } - [HttpGet] + [HttpPost] [Route("jobs/search")] - public async Task>> SearchJobs( - [FromQuery(Name = "q")] string query, - [FromQuery(Name = "d")] int nPreviousDays) + public async Task>> SearchJobs([FromBody] JobQuery jobquery) { - var result = await this.gsEngine.SearchBasicQueryAsync(query, nPreviousDays); - if (result != null) - { - var levels = await this.aiEngine.GetJobLevelAsync(result); - foreach (var level in levels) - { - var job = result.FirstOrDefault(j => j.id == level.Key); - if (job != null) - { - job.tags.Add(level.Value); - } - } - return Ok(result); - } - return StatusCode(500, "Error occurred while searching for jobs."); + return Ok(await jobsRepository.GetJobsFromQuery(jobquery)); } [HttpGet] [Route("jobs/latest")] - public async Task> GetLatestJobsFromScrapper() + public async Task> GetLatestJobsFromDb() { - return Ok(await this.jobsContainer.GetAllLatestJobsAsync()); + return Ok(await this.jobsRepository.GetAllLatestJobsAsync()); + } + + [HttpGet] + [Route("jobs/lastOneDay")] + public async Task> GetLastOneDayJobsFromDb() + { + return Ok(await this.jobsRepository.GetAllJobsInLastOneDay()); } [HttpGet] [Route("jobs/profile/{id}")] public async Task> GetJobById(string id) { - var job = await this.jobsContainer.GetJobByIdAsync(id); + var job = await this.jobsRepository.GetJobByIdAsync(id); if (job != null) { return Ok(job); } return Ok("Not found."); } - - [HttpGet] - [Route("jobs/scrappers")] - public ActionResult> GetAllJobScrappers() - { - // Placeholder implementation for getting all scrappers - return Ok(this.jobscrapperManager.settingsManager.GetAllSettings()); - } - - [HttpPut] - [Route("jobs/scrappers/{id}")] - public ActionResult CreateOrUpdateJobScrapperSettings(string id, [FromBody] ScrapperSettings settings) - { - // Placeholder implementation for updating scrapper settings - return Ok(this.jobscrapperManager.settingsManager.CreateOrUpdateSettings(id, settings)); - } - - [HttpGet] - [Route("jobs/scrappers/{id}")] - public ActionResult GetJobScrapperSettings(string id) - { - // Placeholder implementation for getting scrapper settings - return Ok(this.jobscrapperManager.settingsManager.GetSettingsById(id)); - } - - [HttpGet] - [Route("jobs/scrappers/{id}/trigger")] - public ActionResult TriggerScrapper(string id) - { - // Placeholder implementation for getting scrapper settings - this.jobscrapperManager.RunScrapperByIdAsync(id); - return Ok($"Started scrapper for settings id: {id}"); - } } } \ No newline at end of file diff --git a/src/Backend/Controllers/ScrapperSettingsController.cs b/src/Backend/Controllers/ScrapperSettingsController.cs new file mode 100644 index 0000000..3cbdc4b --- /dev/null +++ b/src/Backend/Controllers/ScrapperSettingsController.cs @@ -0,0 +1,58 @@ +using Common.DatabaseModels; +using Common.Engines; +using Common.Managers; +using Common.Models; +using Common.Models.Public; +using Common.Repositories; +using Microsoft.AspNetCore.Mvc; + +namespace Backend.Controllers +{ + [ApiController] + [Route("api/[controller]")] + public class ScrapperSettingsController : ControllerBase + { + private readonly JobScrapperSettingsManager _settingsManager; + + private readonly ILogger _logger; + + public ScrapperSettingsController( JobScrapperSettingsManager jobScrapperSettingsManager, + ILogger logger) + { + _settingsManager = jobScrapperSettingsManager; + _logger = logger; + } + + [HttpGet] + [Route("jobs/scrappers")] + public async Task>> GetAllJobScrappers() + { + // Placeholder implementation for getting all scrappers + return Ok(await _settingsManager.GetAllSettings()); + } + + [HttpPut] + [Route("jobs/scrappers/{id}")] + public async Task> UpdateJobScrapperSettings(string id, [FromBody] ScrapperSettings settings) + { + // Placeholder implementation for updating scrapper settings + return Ok(await _settingsManager.CreateOrUpdateSettings(id, settings)); + } + + [HttpPost] + [Route("jobs/scrappers/Add")] + public async Task> CreateNewJobScrapperSettings([FromBody] ScrapperSettings settings) + { + // Placeholder implementation for updating scrapper settings + return Ok(await _settingsManager.CreateOrUpdateSettings(string.Empty, settings)); + } + + [HttpGet] + [Route("jobs/scrappers/{id}")] + public async Task> GetJobScrapperSettings(string id) + { + // Placeholder implementation for getting scrapper settings + return Ok(await _settingsManager.GetSettingsById(id)); + } + } +} diff --git a/src/Backend/Filters/IFilter.cs b/src/Backend/Filters/IFilter.cs deleted file mode 100644 index 91a29e5..0000000 --- a/src/Backend/Filters/IFilter.cs +++ /dev/null @@ -1,9 +0,0 @@ -using Common.Models; - -namespace Backend.Filters -{ - public interface IFilter - { - public List ApplyFilterAsync(List problems); - } -} \ No newline at end of file diff --git a/src/Backend/Filters/ProblemFilter.cs b/src/Backend/Filters/ProblemFilter.cs index a4b66f5..4d73c5e 100644 --- a/src/Backend/Filters/ProblemFilter.cs +++ b/src/Backend/Filters/ProblemFilter.cs @@ -1,8 +1,9 @@ +using Common; using Common.Models; namespace Backend.Filters { - public class ProblemFilter : IFilter + public class ProblemFilter : IFilter { private int skip = 0; private int limit = 50; diff --git a/src/Backend/Operations/DataProvider.cs b/src/Backend/Operations/DataProvider.cs index c280dd1..04f80a0 100644 --- a/src/Backend/Operations/DataProvider.cs +++ b/src/Backend/Operations/DataProvider.cs @@ -1,6 +1,7 @@ namespace Backend.Operations { using Backend.Filters; + using Common; using Common.Cache; using Common.Constants; using Common.Models; @@ -15,7 +16,7 @@ public DataProvider([FromKeyedServices(CacheConstants.ProblemCacheKey)] ICache p _logger = logger; } - public async Task> GetProblemsAsync(IFilter? filter = null) + public async Task> GetProblemsAsync(IFilter? filter = null) { var allProblems = await GetAllProblemsAsync(); if (filter != null) diff --git a/src/Backend/Operations/GSEngine.cs b/src/Backend/Operations/GSEngine.cs deleted file mode 100644 index 5d59743..0000000 --- a/src/Backend/Operations/GSEngine.cs +++ /dev/null @@ -1,149 +0,0 @@ -namespace Backend.Operations -{ - using Common.Models; - using Common.Models.Public; - using Microsoft.AspNetCore.Mvc.ModelBinding; - using Newtonsoft.Json; - public class GSEngine - { - private readonly string apiKey; - private readonly string searchEngineId; - private readonly HttpClient httpClient; - private string baseUrl = "https://customsearch.googleapis.com/customsearch/v1"; - private int maxResultsPerSearch = 150; - ILogger logger; - - public GSEngine(IConfiguration configuration, ILogger _logger) - { - this.apiKey = configuration["GoogleSearch:ApiKey"] ?? throw new ArgumentNullException("Google Search API Key is not configured."); - this.searchEngineId = configuration["GoogleSearch:SearchEngineId"] ?? throw new ArgumentNullException("Google Search Engine ID is not configured."); - this.logger = _logger; - this.httpClient = new HttpClient(); - } - - public async Task> SearchQueryAsync(JobScrapperSettings settings) - { - var qsettings = settings.Settings; - var allJobs = new List(); - int startIndex = 1, totalResults = 0; - - var template = $"{this.baseUrl}?key={apiKey}&cx={searchEngineId}&q={Uri.EscapeDataString(qsettings.Query)}"; - template += AddDateRestrictionToQuery(qsettings.lookBackDays); - - if (!string.IsNullOrEmpty(qsettings.ExactTerms)) template += AddExactTermsToQuery(qsettings.ExactTerms); - if (!string.IsNullOrEmpty(qsettings.NegativeTerms)) template += AddNegativeTermToQuery(qsettings.NegativeTerms); - if (!string.IsNullOrEmpty(qsettings.Location)) template += AddClientLocationToQuery(qsettings.Location); - if (!string.IsNullOrEmpty(qsettings.SiteToInclude)) template += AddSiteSearchToQuery(qsettings.SiteToExclude); - if (!string.IsNullOrEmpty(qsettings.SiteToExclude)) template += AddExcludeSiteSearchFromQuery(qsettings.SiteToExclude); - if (!string.IsNullOrEmpty(qsettings.AdditionalSearchterms)) template += AddAdditionalSearchTerms(qsettings.AdditionalSearchterms); - - do - { - var url = template + AddStartIndexToQuery(startIndex); - var res = await SearchRawUrlAsync(url); - if (res == null) - { - logger.LogError("SearchAsync returned null result."); - break; - } - else if (string.IsNullOrEmpty(res.queries.request[0].totalResults) || res.items == null) - { - logger.LogInformation($"No results found for query: {url}"); - break; - } - - foreach (var item in res.items) - { - var job = new ScrappedJob(item, DateTime.UtcNow); - allJobs.Add(job); - } - - totalResults = int.Parse(res.queries.request[0].totalResults); - startIndex += res.queries.request[0].count; - } - while (startIndex < maxResultsPerSearch && startIndex < totalResults); - - this.logger.LogInformation($"Fetched {allJobs.Count} jobs. Total available: {totalResults}. Using url template: {template}"); - - return allJobs; - } - - public async Task> SearchBasicQueryAsync(string query, int nPreviousDays = 1) - { - var qsettings = new Common.Models.Public.QuerySettings - { - query = query, - additionalTerms = "India", - exactTerms = "Software Engineer", - negativeTerms = "Manager", - location = "India", - siteToExclude = "linkedin.com" - }; - var settings = new JobScrapperSettings("basic-search", qsettings, true); - settings.Settings.lookBackDays = nPreviousDays; - return await SearchQueryAsync(settings); - } - - public async Task SearchRawUrlAsync(string url) - { - try - { - var response = await httpClient.GetAsync(url); - response.EnsureSuccessStatusCode(); - var content = await response.Content.ReadAsStringAsync(); - return JsonConvert.DeserializeObject(content); - } - catch (Exception ex) - { - logger.LogError(ex, "Error occurred during Google Search API call."); - } - - return null; - } - - private string AddClientLocationToQuery(string location = "in") - { - return $"&gl={location}"; - } - - private string AddDateRestrictionToQuery(int previousNDays = 1) - { - return $"&dateRestrict=d{previousNDays}"; - } - - private string AddNegativeTermToQuery(string phrase = "manager") - { - return $"&excludeTerms={Uri.EscapeDataString(phrase)}"; - } - - private string AddExactTermsToQuery(string phrase = "Software Engineer") - { - return $"&exactTerms={Uri.EscapeDataString(phrase)}"; - } - - private string AddSiteSearchToQuery(string site = "linkedin.com") - { - return $"&siteSearch={site}&siteSearchFilter=i"; - } - - private string AddExcludeSiteSearchFromQuery(string site = "linkedin.com") - { - return $"&siteSearch={site}&siteSearchFilter=e"; - } - - private string AddSortingToQuery(string sort = "date") - { - return $"&sort={sort}"; - } - - private string AddAdditionalSearchTerms(string terms = "India") - { - return $"&hq={Uri.EscapeDataString(terms)}"; - } - - private string AddStartIndexToQuery(int startIndex = 1) - { - return $"&start={startIndex}"; - } - } -} \ No newline at end of file diff --git a/src/Backend/Operations/JobScrapperManager.cs b/src/Backend/Operations/JobScrapperManager.cs deleted file mode 100644 index 432e148..0000000 --- a/src/Backend/Operations/JobScrapperManager.cs +++ /dev/null @@ -1,53 +0,0 @@ -using Common.Repositories; - -namespace Backend.Operations -{ - public class JobScrapperManager - { - private readonly ILogger logger; - private readonly GSEngine gsEngine; - private readonly AIEngine aiEngine; - private readonly JobsRepository jobsContainer; - public readonly JobScrapperSettingsManager settingsManager; - - - public JobScrapperManager(ILogger logger, GSEngine gsEngine, AIEngine aiEngine, JobScrapperSettingsManager settingsManager, JobsRepository jobsRepo) - { - this.logger = logger; - this.gsEngine = gsEngine; - this.aiEngine = aiEngine; - this.settingsManager = settingsManager; - this.jobsContainer = jobsRepo; - } - - public async Task RunAllScrappersAsync() - { - - } - - public async Task RunScrapperByIdAsync(string id) - { - var settings = this.settingsManager.GetSettingsById(id); - if (settings.Enabled) - { - var scrapper = new JobScrapper(settings, this.gsEngine, this.aiEngine, this.jobsContainer, this.logger); - Task.Run(async () => - { - try - { - await scrapper.RunAsync(); - } - catch (Exception ex) - { - this.logger.LogError($"Error occurred while running scrapper with ID {id}: {ex.Message}"); - } - }); - this.settingsManager.UpdateLastRunTime(id, DateTime.UtcNow); - } - else - { - this.logger.LogWarning($"Scrapper with ID {id} is disabled. Skipping execution."); - } - } - } -} \ No newline at end of file diff --git a/src/Backend/Operations/JobScrapperSettingsManager.cs b/src/Backend/Operations/JobScrapperSettingsManager.cs deleted file mode 100644 index 4771902..0000000 --- a/src/Backend/Operations/JobScrapperSettingsManager.cs +++ /dev/null @@ -1,54 +0,0 @@ -namespace Backend.Operations -{ - using System.Collections.Concurrent; - using System.Globalization; - using System.Reflection.Metadata.Ecma335; - using Common.Models; - - public class JobScrapperSettingsManager - { - private ConcurrentDictionary settingsStore = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); - - public JobScrapperSettingsManager() {} - - public JobScrapperSettings CreateOrUpdateSettings(string id, Common.Models.Public.ScrapperSettings publicSettings) - { - var newSettings = new JobScrapperSettings( - id, - publicSettings.settings, - false); // Initially disabled - - settingsStore.AddOrUpdate(id, newSettings, (key, value) => - { - value.UpdateFromPublicModel(publicSettings); - value.LastUpdated = DateTime.UtcNow; - return value; - }); - - return settingsStore[id]; - } - - public JobScrapperSettings GetSettingsById(string id) - { - if(settingsStore.TryGetValue(id, out var settings)) - { - return settings; - } - return new JobScrapperSettings("NOT FOUND", new Common.Models.Public.QuerySettings(), false); - } - - public List GetAllSettings() - { - return settingsStore.Values.ToList(); - } - - public void UpdateLastRunTime(string id, DateTime runTime) - { - if(settingsStore.TryGetValue(id, out var settings)) - { - settings.LastRunTime = runTime; - settingsStore[id] = settings; - } - } - } -} \ No newline at end of file diff --git a/src/Backend/Program.cs b/src/Backend/Program.cs index 5bace22..8d434c8 100644 --- a/src/Backend/Program.cs +++ b/src/Backend/Program.cs @@ -3,7 +3,9 @@ namespace Backend; using Backend.Operations; using Common.Cache; using Common.Constants; +using Common.Engines; using Common.Factories; +using Common.Managers; using Common.Repositories; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging.ApplicationInsights; @@ -59,12 +61,14 @@ public static void Main(string[] args) { // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle builder.Services.AddEndpointsApiExplorer(); - builder.Services.AddSwaggerGen(); + builder.Services.AddSwaggerGen(c => + { + c.CustomSchemaIds(type => type.FullName); + }); builder.Logging.AddConsole(); } - // Register AppContext as singleton var config = builder.Configuration; #region Register Cosmos related services @@ -98,7 +102,6 @@ public static void Main(string[] args) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); var app = builder.Build(); ILogger logger = app.Logger; diff --git a/src/Backend/appsettings.json b/src/Backend/appsettings.json index 0ea12c8..c1e8513 100644 --- a/src/Backend/appsettings.json +++ b/src/Backend/appsettings.json @@ -11,7 +11,10 @@ "CosmosDbUri": "https://lcw-cosmos.documents.azure.com:443/", "AccountKey": "", "LCProject:DatabaseName": "LeetCodeWrapper", - "LCProject:ContainerName": "Problems" + "LCProject:ContainerName": "Problems", + "JobProject:DatabaseName": "JobDataBase", + "JobProject:ContainerName": "JobDetailsContainer", + "JobProject:ScraperContainerName": "ScraperSettingsContainer" }, "ApplicationInsights": { "LogLevel": { diff --git a/src/Common/Common.csproj b/src/Common/Common.csproj index 844bdc5..91dccc4 100644 --- a/src/Common/Common.csproj +++ b/src/Common/Common.csproj @@ -13,6 +13,9 @@ + + + diff --git a/src/Common/Constants/ConfigurationConstants.cs b/src/Common/Constants/ConfigurationConstants.cs index bb21455..b286deb 100644 --- a/src/Common/Constants/ConfigurationConstants.cs +++ b/src/Common/Constants/ConfigurationConstants.cs @@ -14,6 +14,9 @@ public static class ConfigurationConstants public const string ApplicationSettings = "ApplicationSettings"; public const string LCProjectContainerNameKey = "LCProject:ContainerName"; public const string LCProjectDatabaseNameKey = "LCProject:DatabaseName"; + public const string JobsProjectContainerNameKey = "JobProject:ContainerName"; + public const string JobsScraperSettingsContainerNameKey = "JobProject:ScraperContainerName"; + public const string JobsProjectDatabaseNameKey = "JobProject:DatabaseName"; #endregion } } diff --git a/src/Common/DatabaseModels/JobScrapperSettings.cs b/src/Common/DatabaseModels/JobScrapperSettings.cs new file mode 100644 index 0000000..17f3c6d --- /dev/null +++ b/src/Common/DatabaseModels/JobScrapperSettings.cs @@ -0,0 +1,82 @@ +using System; +using Common.Models.Public; +using PublicSettingsModel = Common.Models.Public.QuerySettings; + +namespace Common.DatabaseModels +{ + public class JobScrapperSettings + { + public string id { get; set; } + + public string settingName { get; set; } + + public bool enabled { get; set; } + + public DateTime lastUpdated { get; set; } + + public DateTime lastRunTime { get; set; } + + public int runIntervalInMinutes { get; set; } + + public QuerySettings settings { get; set; } + + public JobScrapperSettings(string id, + string settingName, + int? runIntervalsInMinutes, + PublicSettingsModel settings, + bool enabled = false) + { + this.id = id; + this.settingName = settingName; + this.enabled = enabled; + this.lastUpdated = DateTime.UtcNow; + this.lastRunTime = DateTime.MinValue; + this.runIntervalInMinutes = Math.Min(60, runIntervalsInMinutes ?? 60); + this.settings = new QuerySettings(settings); + } + + public void UpdateFromPublicModel(ScrapperSettings publicSettings) + { + if (publicSettings == null) throw new ArgumentNullException(nameof(publicSettings)); + + this.enabled = publicSettings.enabled; + this.runIntervalInMinutes = publicSettings.runIntervalInMinutes; + this.settings = new QuerySettings(publicSettings.settings); + this.lastUpdated = DateTime.UtcNow; + // keep SettingName unchanged unless public model provides one + if (!string.IsNullOrWhiteSpace(publicSettings.name)) + { + this.settingName = publicSettings.name; + } + } + + public ScrapperSettings ToPublicModel() + { + return new ScrapperSettings + { + id = this.id, + name = this.settingName, + enabled = this.enabled, + lastUpdated = this.lastUpdated, + lastRunTime = this.lastRunTime, + runIntervalInMinutes = this.runIntervalInMinutes, + settings = new PublicSettingsModel + { + query = this.settings.query, + locations = this.settings.locations, + sitesToInclude = this.settings.sitesToInclude, + sitesToExclude = this.settings.sitesToExclude, + exactTerms = this.settings.exactTerms, + negativeTerms = this.settings.negativeTerms, + additionalTerms = this.settings.additionalSearchterms, + lookBackDays = this.settings.lookBackDays + } + }; + } + + public QuerySettings GetQuerySettings() + { + return this.settings; + } + } +} \ No newline at end of file diff --git a/src/Common/Models/ProblemSchema.cs b/src/Common/DatabaseModels/ProblemSchema.cs similarity index 55% rename from src/Common/Models/ProblemSchema.cs rename to src/Common/DatabaseModels/ProblemSchema.cs index edbc20d..137bb81 100644 --- a/src/Common/Models/ProblemSchema.cs +++ b/src/Common/DatabaseModels/ProblemSchema.cs @@ -1,30 +1,32 @@ -namespace Common.Models +using Common.Models; + +namespace Common.DatabaseModels { public class ProblemSchema { public ProblemSchema() { } public ProblemSchema(ProblemSchema ps) { - this.id = ps.id; - this.title = ps.title; - this.url = ps.url; - this.difficulty = ps.difficulty; - this.acceptance = ps.acceptance; - this.frequency = ps.frequency; - this.companyList = new List>>(); - this.metadataList = new List>(); + id = ps.id; + title = ps.title; + url = ps.url; + difficulty = ps.difficulty; + acceptance = ps.acceptance; + frequency = ps.frequency; + companyList = new List>>(); + metadataList = new List>(); } public ProblemSchema(Problem p) { - this.id = p.id; - this.title = p.title; - this.url = p.url; - this.difficulty = p.difficulty; - this.acceptance = p.acceptance; - this.frequency = p.frequency; - this.companyList = p.companies.Select(kv => new KeyValuePair>(kv.Key, kv.Value.ToList())).ToList(); - this.metadataList = p.metadata.Select(kv => new KeyValuePair(kv.Key, kv.Value)).ToList(); + id = p.id; + title = p.title; + url = p.url; + difficulty = p.difficulty; + acceptance = p.acceptance; + frequency = p.frequency; + companyList = p.companies.Select(kv => new KeyValuePair>(kv.Key, kv.Value.ToList())).ToList(); + metadataList = p.metadata.Select(kv => new KeyValuePair(kv.Key, kv.Value)).ToList(); } public string id { get; set; } = string.Empty; diff --git a/src/Common/DatabaseModels/QuerySettings.cs b/src/Common/DatabaseModels/QuerySettings.cs new file mode 100644 index 0000000..c03571a --- /dev/null +++ b/src/Common/DatabaseModels/QuerySettings.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using PublicSettingsModel = Common.Models.Public.QuerySettings; + +namespace Common.DatabaseModels +{ + + public class QuerySettings + { + public string query { get; set; } + public List locations { get; set; } + public List sitesToInclude { get; set; } + public List sitesToExclude { get; set; } + public List exactTerms { get; set; } + public List negativeTerms { get; set; } + public int lookBackDays { get; set; } = 1; + public List additionalSearchterms { get; set; } + + public QuerySettings(PublicSettingsModel qs) + { + query = qs.query; + locations = qs.locations; + sitesToInclude = qs.sitesToInclude; + sitesToExclude = qs.sitesToExclude; + exactTerms = qs.exactTerms; + negativeTerms = qs.negativeTerms; + additionalSearchterms = qs.additionalTerms; + } + } +} diff --git a/src/Common/Models/ScrappedJob.cs b/src/Common/DatabaseModels/ScrappedJob.cs similarity index 54% rename from src/Common/Models/ScrappedJob.cs rename to src/Common/DatabaseModels/ScrappedJob.cs index 2758490..f334826 100644 --- a/src/Common/Models/ScrappedJob.cs +++ b/src/Common/DatabaseModels/ScrappedJob.cs @@ -1,4 +1,6 @@ -namespace Common.Models +using Common.Models; + +namespace Common.DatabaseModels { public class ScrappedJob { @@ -9,23 +11,27 @@ public class ScrappedJob public string description { get; set; } public string link { get; set; } public DateTime scrappedTime { get; set; } + public DateTime JobPostedTime { get; set; } + public string companyName { get; set; } + public string jobType { get; set; } + public string location { get; set; } public List tags { get; set; } = new List(); public ScrappedJob() { } public ScrappedJob(Item item, DateTime scrappedTime) { - this.title = item.title; - this.displayLink = item.displayLink; - this.snippet = item.snippet; - this.link = item.link; - this.id = GenerateHashId(item.link, item.title, item.displayLink); + title = item.title; + displayLink = item.displayLink; + snippet = item.snippet; + link = item.link; + id = GenerateHashId(item.link, item.title, item.displayLink); this.scrappedTime = scrappedTime; - this.description = "NA"; + description = "NA"; } private string GenerateHashId(string v1, string v2, string v3) { - return Common.Helper.FastHashId.GenerateHashId(v1, v2, v3); + return Helper.FastHashId.GenerateHashId(v1, v2, v3); } } } \ No newline at end of file diff --git a/src/Backend/Operations/AIEngine.cs b/src/Common/Engines/AIEngine.cs similarity index 63% rename from src/Backend/Operations/AIEngine.cs rename to src/Common/Engines/AIEngine.cs index 631336c..ee3fa22 100644 --- a/src/Backend/Operations/AIEngine.cs +++ b/src/Common/Engines/AIEngine.cs @@ -1,4 +1,4 @@ -namespace Backend.Operations +namespace Common.Engines { using Azure; using Azure.AI; @@ -7,8 +7,10 @@ namespace Backend.Operations using Azure.AI.Projects; using Azure.AI.Agents.Persistent; using System.Diagnostics; - using Common.Models; using Newtonsoft.Json; + using Microsoft.Extensions.Logging; + using Microsoft.Extensions.Configuration; + using Common.DatabaseModels; public class AIEngine { @@ -79,51 +81,52 @@ private async Task GetResponseInternalAsync(string input) { if (!IsReady()) { - this.logger.LogError($"AIEngine is not properly initialized. Given input: {input}"); - throw new InvalidOperationException("AIEngine is not properly initialized."); + logger.LogError($"AIEngine is not properly initialized. Input: {input}"); + throw new InvalidOperationException("AIEngine not initialized."); } - PersistentAgentThread thread = agentsClient.Threads.CreateThread(); - - PersistentThreadMessage messageResponse = agentsClient.Messages.CreateMessage( - thread.Id, - MessageRole.User, - input); + var threadResponse = await agentsClient.Threads.CreateThreadAsync(); + var thread = threadResponse.Value; - ThreadRun run = agentsClient.Runs.CreateRun( - thread.Id, - agent.Id); - - // Poll until the run reaches a terminal status - do - { - await Task.Delay(TimeSpan.FromMilliseconds(500)); - run = agentsClient.Runs.GetRun(thread.Id, run.Id); - } - while (run.Status == RunStatus.Queued - || run.Status == RunStatus.InProgress); - if (run.Status != RunStatus.Completed) + try { - this.logger.LogError($"Run failed or was canceled. ThreadId: {thread.Id} Last error: {run.LastError?.Message}"); - throw new InvalidOperationException($"Run failed or was canceled: {run.LastError?.Message}"); - } + await agentsClient.Messages.CreateMessageAsync(thread.Id, MessageRole.User, input); + var runResponse = await agentsClient.Runs.CreateRunAsync(thread.Id, agent.Id); + var run = runResponse.Value; - Pageable messages = agentsClient.Messages.GetMessages( - thread.Id, order: ListSortOrder.Ascending); - - string response = string.Empty; - PersistentThreadMessage lastThreadMessage = messages.Last(); + // Poll until terminal state + do + { + await Task.Delay(500); + run = await agentsClient.Runs.GetRunAsync(thread.Id, run.Id); + } + while (run.Status == RunStatus.Queued || run.Status == RunStatus.InProgress); - foreach (MessageContent contentItem in lastThreadMessage.ContentItems) - { - if (contentItem is MessageTextContent textItem) + if (run.Status != RunStatus.Completed) { - response += textItem.Text; + logger.LogError($"Run failed. ThreadId={thread.Id}, Error={run.LastError?.Message}"); + throw new InvalidOperationException($"Run failed: {run.LastError?.Message}"); + } + + // Fetch all messages in ascending order + var messages = agentsClient.Messages.GetMessagesAsync(thread.Id, order: ListSortOrder.Ascending); + + string response = string.Empty; + PersistentThreadMessage lastThreadMessage = messages.ToBlockingEnumerable().Last(); + foreach (MessageContent contentItem in lastThreadMessage.ContentItems) + { + if (contentItem is MessageTextContent textItem) + { + response += textItem.Text; + } } - } - agentsClient.Threads.DeleteThread(thread.Id); - return response; + return response; + } + finally + { + await agentsClient.Threads.DeleteThreadAsync(thread.Id); + } } - } + } } \ No newline at end of file diff --git a/src/Common/Engines/GSEngine.cs b/src/Common/Engines/GSEngine.cs new file mode 100644 index 0000000..231c96b --- /dev/null +++ b/src/Common/Engines/GSEngine.cs @@ -0,0 +1,238 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Common.DatabaseModels; +using Common.Models; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace Common.Engines +{ + public class GSEngine + { + private readonly string apiKey; + private readonly string searchEngineId; + private readonly HttpClient httpClient; + private string baseUrl = "https://customsearch.googleapis.com/customsearch/v1"; + private int maxResultsPerSearch = 150; + private readonly ILogger logger; + + public GSEngine(IConfiguration configuration, ILogger _logger) + { + this.apiKey = configuration["GoogleSearch:ApiKey"] ?? throw new ArgumentNullException("Google Search API Key is not configured."); + this.searchEngineId = configuration["GoogleSearch:SearchEngineId"] ?? throw new ArgumentNullException("Google Search Engine ID is not configured."); + this.logger = _logger; + this.httpClient = new HttpClient(); + } + + public async Task> SearchQueryAsync(JobScrapperSettings settings) + { + if (settings == null) throw new ArgumentNullException(nameof(settings)); + + var qsettings = settings.GetQuerySettings() ?? throw new InvalidOperationException("Query settings cannot be null."); + var allJobs = new List(); + int startIndex = 1; + int totalResults = 0; + + var sb = new StringBuilder(); + sb.Append($"{this.baseUrl}?key={apiKey}&cx={searchEngineId}"); + + // base query + var baseQuery = qsettings.query ?? string.Empty; + sb.Append($"&q={Uri.EscapeDataString(baseQuery)}"); + + // date restriction + if (qsettings.lookBackDays > 0) + { + sb.Append(AddDateRestrictionToQuery(qsettings.lookBackDays)); + } + + // Exact terms (join list if provided) + if (qsettings.exactTerms != null && qsettings.exactTerms.Any()) + { + var exact = string.Join(" ", qsettings.exactTerms.Where(s => !string.IsNullOrWhiteSpace(s))); + if (!string.IsNullOrWhiteSpace(exact)) sb.Append(AddExactTermsToQuery(exact)); + } + + // Negative terms + if (qsettings.negativeTerms != null && qsettings.negativeTerms.Any()) + { + var neg = string.Join(" ", qsettings.negativeTerms.Where(s => !string.IsNullOrWhiteSpace(s))); + if (!string.IsNullOrWhiteSpace(neg)) sb.Append(AddNegativeTermToQuery(neg)); + } + + // Location - use first location if present (api uses gl for country) + if (qsettings.locations != null && qsettings.locations.Any() && !string.IsNullOrWhiteSpace(qsettings.locations.First())) + { + sb.Append(AddClientLocationToQuery(qsettings.locations.First())); + } + + // Site include / exclude - use first for siteSearch (API supports one siteSearch parameter) + if (qsettings.sitesToInclude != null && qsettings.sitesToInclude.Any() && !string.IsNullOrWhiteSpace(qsettings.sitesToInclude.First())) + { + sb.Append(AddSiteSearchToQuery(qsettings.sitesToInclude.First())); + } + else if (qsettings.sitesToExclude != null && qsettings.sitesToExclude.Any() && !string.IsNullOrWhiteSpace(qsettings.sitesToExclude.First())) + { + // prefer include if present; otherwise exclude + sb.Append(AddExcludeSiteSearchFromQuery(qsettings.sitesToExclude.First())); + } + + // Additional terms (hq) + if (qsettings.additionalSearchterms != null && qsettings.additionalSearchterms.Any()) + { + var add = string.Join(" ", qsettings.additionalSearchterms.Where(s => !string.IsNullOrWhiteSpace(s))); + if (!string.IsNullOrWhiteSpace(add)) sb.Append(AddadditionalSearchterms(add)); + } + + var template = sb.ToString(); + + do + { + var url = template + AddStartIndexToQuery(startIndex); + var res = await SearchRawUrlAsync(url); + if (res == null) + { + logger.LogError("SearchRawUrlAsync returned null for url: {url}", url); + break; + } + + // No items => stop + if (res.items == null || res.items.Count == 0) + { + logger.LogInformation("No items returned for url: {url}", url); + break; + } + + foreach (var item in res.items) + { + try + { + var job = new ScrappedJob(item, DateTime.UtcNow); + allJobs.Add(job); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Skipping item due to processing error."); + } + } + + // Determine total results + if (!string.IsNullOrWhiteSpace(res.searchInformation?.totalResults)) + { + if (!int.TryParse(res.searchInformation.totalResults, out totalResults)) + { + // try fallback to queries.request[0].totalResults + var reqTotal = res.queries?.request?.FirstOrDefault()?.totalResults; + if (!int.TryParse(reqTotal, out totalResults)) totalResults = int.MaxValue; + } + } + else + { + var reqTotal = res.queries?.request?.FirstOrDefault()?.totalResults; + if (!int.TryParse(reqTotal, out totalResults)) totalResults = int.MaxValue; + } + + // Advance to next page if present + if (res.queries?.nextPage != null && res.queries.nextPage.Count > 0) + { + var next = res.queries.nextPage[0]; + // Use next.startIndex if present; otherwise increment by count + if (next.startIndex > 0) + { + startIndex = next.startIndex; + } + else + { + var count = res.queries.request?.FirstOrDefault()?.count ?? res.items.Count; + if (count <= 0) break; + startIndex += count; + } + } + else + { + // no next page -> stop + break; + } + + // safety: prevent infinite looping + if (startIndex <= 0 || startIndex > maxResultsPerSearch) break; + } + while (startIndex <= maxResultsPerSearch && (totalResults == 0 || startIndex <= totalResults)); + + this.logger.LogInformation("Fetched {count} jobs. Total available (approx): {total}. Url template: {template}", allJobs.Count, totalResults, template); + return allJobs; + } + + public async Task SearchRawUrlAsync(string url) + { + try + { + var response = await httpClient.GetAsync(url); + if (!response.IsSuccessStatusCode) + { + logger.LogWarning("Google Search API returned status {status} for url {url}", response.StatusCode, url); + return null; + } + + var content = await response.Content.ReadAsStringAsync(); + return JsonConvert.DeserializeObject(content); + } + catch (Exception ex) + { + logger.LogError(ex, "Error occurred during Google Search API call."); + } + + return null; + } + + private string AddClientLocationToQuery(string location = "in") + { + return $"&gl={Uri.EscapeDataString(location)}"; + } + + private string AddDateRestrictionToQuery(int previousNDays = 1) + { + return $"&dateRestrict=d{previousNDays}"; + } + + private string AddNegativeTermToQuery(string phrase = "manager") + { + return $"&excludeTerms={Uri.EscapeDataString(phrase)}"; + } + + private string AddExactTermsToQuery(string phrase = "Software Engineer") + { + return $"&exactTerms={Uri.EscapeDataString(phrase)}"; + } + + private string AddSiteSearchToQuery(string site = "linkedin.com") + { + return $"&siteSearch={Uri.EscapeDataString(site)}&siteSearchFilter=i"; + } + + private string AddExcludeSiteSearchFromQuery(string site = "linkedin.com") + { + return $"&siteSearch={Uri.EscapeDataString(site)}&siteSearchFilter=e"; + } + + private string AddSortingToQuery(string sort = "date") + { + return $"&sort={Uri.EscapeDataString(sort)}"; + } + + private string AddadditionalSearchterms(string terms = "India") + { + return $"&hq={Uri.EscapeDataString(terms)}"; + } + + private string AddStartIndexToQuery(int startIndex = 1) + { + return $"&start={startIndex}"; + } + } +} \ No newline at end of file diff --git a/src/Common/Factories/CosmosContainerFactory.cs b/src/Common/Factories/CosmosContainerFactory.cs index 2253602..f54b062 100644 --- a/src/Common/Factories/CosmosContainerFactory.cs +++ b/src/Common/Factories/CosmosContainerFactory.cs @@ -3,6 +3,7 @@ using Common.Models.Miscellaneous; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; namespace Common.Factories { @@ -12,33 +13,30 @@ public class CosmosContainerFactory : ICosmosContainerFactory private readonly IConfiguration _configuration; - public CosmosContainerFactory(CosmosClient cosmosClient, IConfiguration configuration) + private readonly ILogger _logger; + public CosmosContainerFactory(CosmosClient cosmosClient, + IConfiguration configuration, + ILogger logger) { _cosmosClient = cosmosClient; _configuration = configuration; + _logger = logger; } public Container GetContainer(CosmosContainerEnum container) { var containerDetails = LoadContainerDetails(); - string dbId; - string containerId; - switch (container) + + if(!containerDetails.ContainsKey(container)) { - case CosmosContainerEnum.ProblemsContainer: - dbId = containerDetails[container].DatabaseName; - containerId = containerDetails[container].ContainerName; - break; - case CosmosContainerEnum.JobsContainer: - dbId = "JobDataBase"; - containerId = "JobDetailsContainer"; - break; - default: - throw new ArgumentOutOfRangeException(nameof(container), container, null); + _logger.LogError("Container details not found for container: {Container}", container); + throw new ArgumentOutOfRangeException(nameof(container), container, null); } - - var db = _cosmosClient.GetDatabase(dbId); - return db.GetContainer(containerId); + + var databaseName = containerDetails[container].DatabaseName; + var containerName = containerDetails[container].ContainerName; + var dbInstnace = _cosmosClient.GetDatabase(databaseName); + return dbInstnace.GetContainer(containerName); } private Dictionary LoadContainerDetails() @@ -49,6 +47,14 @@ private Dictionary LoadContainerDetails() { CosmosContainerEnum.ProblemsContainer, new ContainerDetails(config[ConfigurationConstants.LCProjectDatabaseNameKey], config[ConfigurationConstants.LCProjectContainerNameKey]) + }, + { + CosmosContainerEnum.JobsContainer, + new ContainerDetails(config[ConfigurationConstants.JobsProjectDatabaseNameKey], config[ConfigurationConstants.JobsProjectContainerNameKey]) + }, + { + CosmosContainerEnum.ScrapperSettingsContainer, + new ContainerDetails(config[ConfigurationConstants.JobsProjectDatabaseNameKey], config[ConfigurationConstants.JobsScraperSettingsContainerNameKey]) } }; } diff --git a/src/Common/IFilter.cs b/src/Common/IFilter.cs new file mode 100644 index 0000000..0a90e8c --- /dev/null +++ b/src/Common/IFilter.cs @@ -0,0 +1,7 @@ +namespace Common +{ + public interface IFilter + { + public List ApplyFilterAsync(List entities); + } +} \ No newline at end of file diff --git a/src/Backend/Operations/JobScrapper.cs b/src/Common/Managers/JobScrapper.cs similarity index 79% rename from src/Backend/Operations/JobScrapper.cs rename to src/Common/Managers/JobScrapper.cs index 349d9e7..f7f9a45 100644 --- a/src/Backend/Operations/JobScrapper.cs +++ b/src/Common/Managers/JobScrapper.cs @@ -1,23 +1,30 @@ -namespace Backend.Operations +namespace Common.Managers + { - using Common.Models; + using Common.DatabaseModels; + using Common.Engines; using Common.Repositories; + using Microsoft.Extensions.Logging; public class JobScrapper { private JobScrapperSettings settings; private GSEngine gsEngine; private AIEngine aiEngine; - private JobsRepository jobsContainer; + private JobsRepository jobsRepository; private ILogger logger; - public JobScrapper(JobScrapperSettings settings, GSEngine gsEngine, AIEngine aiEngine, JobsRepository jobsRepo, ILogger logger) + public JobScrapper(GSEngine gsEngine, AIEngine aiEngine, JobsRepository jobsRepo, ILogger logger) { this.logger = logger; this.gsEngine = gsEngine; this.aiEngine = aiEngine; + this.jobsRepository = jobsRepo; + } + + public void ConfigureSettings(JobScrapperSettings settings) + { this.settings = settings; - this.jobsContainer = jobsRepo; } public async Task RunAsync() @@ -49,7 +56,7 @@ public async Task RunAsync() foreach (var job in searchResults) { - var success = await this.jobsContainer.CreateOrUpdateJobAsync(job); + var success = await this.jobsRepository.CreateIfNotExistsAsync(job); if (!success) { this.logger.LogError($"Failed to push job {job.id} to JobsRepository."); diff --git a/src/Common/Managers/JobScrapperSettingsManager.cs b/src/Common/Managers/JobScrapperSettingsManager.cs new file mode 100644 index 0000000..3d434f7 --- /dev/null +++ b/src/Common/Managers/JobScrapperSettingsManager.cs @@ -0,0 +1,93 @@ +namespace Common.Managers +{ + using Common.DatabaseModels; + using Common.Enums; + using Common.Factories; + using Common.Models.Public; + using Microsoft.Azure.Cosmos; + using Microsoft.Extensions.Logging; + public class JobScrapperSettingsManager + { + private readonly Container _scrapperSettingsContainer; + private readonly ILogger _logger; + + public JobScrapperSettingsManager(ICosmosContainerFactory cosmosContainerFactory, + ILogger logger) + { + _scrapperSettingsContainer = cosmosContainerFactory.GetContainer(CosmosContainerEnum.ScrapperSettingsContainer); + _logger = logger; + } + + public async Task CreateOrUpdateSettings(string id, ScrapperSettings publicSettings) + { + if(publicSettings == null) + { + throw new ArgumentNullException(nameof(publicSettings), "Public settings cannot be null"); + } + + var settingsInDb = _scrapperSettingsContainer.GetItemQueryIterator($"SELECT TOP 1* from ScraperSettingsContainer where Id = {id}"); + + int count = 0; + var existingSettingsList = new List(); + var returnSettings = default(JobScrapperSettings); + while (settingsInDb.HasMoreResults) + { + var response = await settingsInDb.ReadNextAsync(); + existingSettingsList.AddRange(response); + } + + if(count > 0) + { + var existingSettings = existingSettingsList[0]; + existingSettings.UpdateFromPublicModel(publicSettings); + await _scrapperSettingsContainer.ReplaceItemAsync( + existingSettings, + existingSettings.id + ); + returnSettings = existingSettings; + } + else + { + id = Guid.NewGuid().ToString(); + returnSettings = await _scrapperSettingsContainer.CreateItemAsync( + new JobScrapperSettings( + id, + publicSettings.name, + publicSettings.runIntervalInMinutes, + publicSettings.settings, + true) + ); + } + + return returnSettings; + } + + public async Task GetSettingsById(string id) + { + var setting = await _scrapperSettingsContainer.ReadItemAsync( + id, + new PartitionKey(id) + ); + + if(setting == null) + { + _logger.LogError($"No JobScrapperSettings found with id: {id}"); + throw new KeyNotFoundException($"No JobScrapperSettings found with id: {id}"); + } + + return setting; + } + + public async Task> GetAllSettings() + { + var settingsInDb = _scrapperSettingsContainer.GetItemQueryIterator($"SELECT * from ScraperSettingsContainer"); + var allSettings = new List(); + while (settingsInDb.HasMoreResults) + { + var response = await settingsInDb.ReadNextAsync(); + allSettings.AddRange(response); + } + return allSettings; + } + } +} \ No newline at end of file diff --git a/src/Common/Models/JobScrapperSettings.cs b/src/Common/Models/JobScrapperSettings.cs deleted file mode 100644 index d97b7a4..0000000 --- a/src/Common/Models/JobScrapperSettings.cs +++ /dev/null @@ -1,83 +0,0 @@ -namespace Common.Models -{ - public class JobScrapperSettings - { - public string Id { get; set; } - public bool Enabled { get; set; } - public DateTime LastUpdated { get; set; } - public DateTime LastRunTime { get; set; } - public int RunIntervalInHours { get; set; } - public QuerySettings Settings { get; set; } - - public JobScrapperSettings(string id, Models.Public.QuerySettings settings, bool enabled = false) - { - this.Id = id; - this.Enabled = enabled; - this.LastUpdated = DateTime.UtcNow; - this.LastRunTime = DateTime.MinValue; - this.RunIntervalInHours = 24; // Default to daily runs - this.Settings = new Models.QuerySettings(settings); - } - - public string GetQueryParameters() - { - return string.Empty; - } - - public void UpdateFromPublicModel(Models.Public.ScrapperSettings publicSettings) - { - this.Enabled = publicSettings.enabled; - this.RunIntervalInHours = publicSettings.runIntervalInHours; - this.Settings = new Models.QuerySettings(publicSettings.settings); - } - - public Models.Public.ScrapperSettings ToPublicModel() - { - return new Models.Public.ScrapperSettings - { - id = this.Id, - enabled = this.Enabled, - lastUpdated = this.LastUpdated, - lastRunTime = this.LastRunTime, - runIntervalInHours = this.RunIntervalInHours, - settings = new Models.Public.QuerySettings - { - query = this.Settings.Query, - location = this.Settings.Location, - siteToInclude = this.Settings.SiteToInclude, - siteToExclude = this.Settings.SiteToExclude, - exactTerms = this.Settings.ExactTerms, - negativeTerms = this.Settings.NegativeTerms - } - }; - } - - public override string ToString() - { - return $"JobScrapperSettings(Id={Id}, Enabled={Enabled}, LastUpdated={LastUpdated}, LastRunTime={LastRunTime}, RunIntervalInHours={RunIntervalInHours}, Settings=[Query={Settings.Query}, Location={Settings.Location}])"; - } - } - - public class QuerySettings - { - public string Query { get; set; } - public string Location { get; set; } - public string SiteToInclude { get; set; } - public string SiteToExclude { get; set; } - public string ExactTerms { get; set; } - public string NegativeTerms { get; set; } - public int lookBackDays = 1; - public string AdditionalSearchterms { get; set; } - - public QuerySettings(Models.Public.QuerySettings qs) - { - this.Query = qs.query; - this.Location = qs.location; - this.SiteToInclude = qs.siteToInclude; - this.SiteToExclude = qs.siteToExclude; - this.ExactTerms = qs.exactTerms; - this.NegativeTerms = qs.negativeTerms; - this.AdditionalSearchterms = qs.additionalTerms; - } - } -} \ No newline at end of file diff --git a/src/Common/Models/Problem.cs b/src/Common/Models/Problem.cs index 27b6f19..714968f 100644 --- a/src/Common/Models/Problem.cs +++ b/src/Common/Models/Problem.cs @@ -1,3 +1,5 @@ +using Common.DatabaseModels; + namespace Common.Models { public enum Difficulty diff --git a/src/Common/Models/Public/QuerySettings.cs b/src/Common/Models/Public/QuerySettings.cs new file mode 100644 index 0000000..682021b --- /dev/null +++ b/src/Common/Models/Public/QuerySettings.cs @@ -0,0 +1,15 @@ +namespace Common.Models.Public +{ + + public class QuerySettings + { + public string query { get; set; } + public List locations { get; set; } + public List sitesToInclude { get; set; } + public List sitesToExclude { get; set; } + public List exactTerms { get; set; } + public List negativeTerms { get; set; } + public List additionalTerms { get; set; } + public int lookBackDays { get; set; } + } +} diff --git a/src/Common/Models/Public/ScrapperSettings.cs b/src/Common/Models/Public/ScrapperSettings.cs index 1a6a320..85d424b 100644 --- a/src/Common/Models/Public/ScrapperSettings.cs +++ b/src/Common/Models/Public/ScrapperSettings.cs @@ -3,21 +3,11 @@ namespace Common.Models.Public public class ScrapperSettings { public string id { get; set; } + public string name { get; set; } public bool enabled { get; set; } public DateTime lastUpdated { get; set; } public DateTime lastRunTime { get; set; } - public int runIntervalInHours { get; set; } + public int runIntervalInMinutes { get; set; } public QuerySettings settings { get; set; } } - - public class QuerySettings - { - public string query { get; set; } - public string location { get; set; } - public string siteToInclude { get; set; } - public string siteToExclude { get; set; } - public string exactTerms { get; set; } - public string negativeTerms { get; set; } - public string additionalTerms { get; set; } - } } \ No newline at end of file diff --git a/src/Common/Queries/JobQuery.cs b/src/Common/Queries/JobQuery.cs new file mode 100644 index 0000000..3bcce34 --- /dev/null +++ b/src/Common/Queries/JobQuery.cs @@ -0,0 +1,12 @@ +namespace Common.Queries +{ + public class JobQuery + { + public string JobType { get; set; } // Software Engineer, Data Scientist, etc. + public DateTime StartDate { get; set; } = DateTime.UtcNow; // Start date for the job posting + public DateTime EndDate { get; set; } = DateTime.UtcNow; // End date for the job posting + public List Companies { get; set; } // List of companies to filter + public List Locations { get; set; } // List of locations to filter + public string JobLevel { get; set; } // Entry Level, Mid Level, Senior Level, etc. + } +} diff --git a/src/Common/Repositories/JobScrapperSettingsRepository.cs b/src/Common/Repositories/JobScrapperSettingsRepository.cs new file mode 100644 index 0000000..8b2edb0 --- /dev/null +++ b/src/Common/Repositories/JobScrapperSettingsRepository.cs @@ -0,0 +1,53 @@ +using Common.DatabaseModels; +using Common.Enums; +using Common.Factories; +using Common.Managers; +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Common.Repositories +{ + public class JobScrapperSettingsRepository + { + private readonly Container _scrapperSettingsContainer; + private readonly ILogger _logger; + + public JobScrapperSettingsRepository(ICosmosContainerFactory cosmosContainerFactory, + ILogger logger) + { + _scrapperSettingsContainer = cosmosContainerFactory.GetContainer(CosmosContainerEnum.ScrapperSettingsContainer); + _logger = logger; + } + + public async Task> GetAllSettings() + { + var settingsInDb = _scrapperSettingsContainer.GetItemQueryIterator($"SELECT * from JobScrapperSettings"); + var allSettings = new List(); + while (settingsInDb.HasMoreResults) + { + var response = await settingsInDb.ReadNextAsync(); + allSettings.AddRange(response); + } + return allSettings; + } + + public async Task UpdateSettingsAsync(string id, JobScrapperSettings jobSetting) + { + try + { + await _scrapperSettingsContainer.UpsertItemAsync(jobSetting, new PartitionKey(id)); + _logger.LogInformation($"Successfully updated JobScrapperSettings with id: {id}"); + } + catch (Exception ex) + { + _logger.LogError($"Error updating JobScrapperSettings with id: {id}. Exception: {ex.Message}"); + throw; + } + } + } +} diff --git a/src/Common/Repositories/JobsRepository.cs b/src/Common/Repositories/JobsRepository.cs index f95d4b8..3086fdb 100644 --- a/src/Common/Repositories/JobsRepository.cs +++ b/src/Common/Repositories/JobsRepository.cs @@ -1,10 +1,12 @@ namespace Common.Repositories { + using Common.DatabaseModels; using Common.Enums; using Common.Factories; - using Common.Models; + using Common.Queries; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; + using System.Net; public class JobsRepository { @@ -50,22 +52,41 @@ public async Task GetJobByIdAsync(string id) } } - public async Task CreateOrUpdateJobAsync(ScrappedJob job) + /// + /// Create the item only if it does not already exist using a single DB call. + /// Returns true if the item was created, false if it already existed. + /// + public async Task CreateIfNotExistsAsync(ScrappedJob job) { + if (job == null) throw new ArgumentNullException(nameof(job)); try { - // TODO: Do async inserts for faster performance - var res = await this.jobsContainer.UpsertItemAsync(job); + var requestOptions = new ItemRequestOptions + { + // Instruct Cosmos to only create if the item does not exist. + // SDK will translate this to an If-None-Match header. + IfNoneMatchEtag = "*" + }; + + var response = await this.jobsContainer.CreateItemAsync(job, new PartitionKey(job.id), requestOptions); + // Created successfully + this.logger.LogInformation("Created job {id} in Cosmos DB. RU charge: {ru}", job.id, response.RequestCharge); + return true; } - catch (Exception ex) + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.PreconditionFailed || ex.StatusCode == HttpStatusCode.Conflict) { - this.logger.LogError($"Failed to push job: {job.id} to container. Ex: {ex}"); + // Item already exists (server enforces the If-None-Match precondition). + this.logger.LogInformation("Job {id} already exists. Skipping create.", job.id); return false; } - - return true; + catch (Exception ex) + { + this.logger.LogError(ex, "Failed to create job {id} in Cosmos DB.", job.id); + throw; + } } + private async Task> QueryJobsAsync(string query) { var queryDefinition = new QueryDefinition(query); @@ -79,5 +100,99 @@ private async Task> QueryJobsAsync(string query) this.logger.LogInformation($"Retrieved {results.Count} jobs from Cosmos DB. Query: {query}"); return results; } + private async Task> QueryJobsAsync(QueryDefinition queryDefinition) + { + var queryResultSetIterator = jobsContainer.GetItemQueryIterator(queryDefinition); + List results = new List(); + while (queryResultSetIterator.HasMoreResults) + { + var response = await queryResultSetIterator.ReadNextAsync(); + results.AddRange(response); + } + this.logger.LogInformation($"Retrieved {results.Count} jobs from Cosmos DB."); + return results; + } + + public async Task> GetJobsFromQuery(JobQuery jobquery) + { + if (jobquery == null) throw new ArgumentNullException(nameof(jobquery)); + + var sql = "SELECT * FROM c WHERE 1=1"; + var qd = new QueryDefinition(sql); + + // JobType: search title or tags + if (!string.IsNullOrWhiteSpace(jobquery.JobType)) + { + qd = qd.WithParameter("@jobType", jobquery.JobType); + sql += " AND CONTAINS(c.jobType, @jobType, true)"; + } + + // Companies (list) + if (jobquery.Companies != null && jobquery.Companies.Count > 0) + { + var companyConditions = new List(); + for (int i = 0; i < jobquery.Companies.Count; i++) + { + var param = $"@company{i}"; + qd = qd.WithParameter(param, jobquery.Companies[i]); + companyConditions.Add($"c.companyName = {param}"); + } + sql += " AND (" + string.Join(" OR ", companyConditions) + ")"; + } + + // Locations: fallback to searching in displayLink, snippet or description + if (jobquery.Locations != null && jobquery.Locations.Count > 0) + { + var locationConditions = new List(); + for (int i = 0; i < jobquery.Locations.Count; i++) + { + var param = $"@location{i}"; + qd = qd.WithParameter(param, jobquery.Locations[i]); + locationConditions.Add($"CONTAINS(c.location, {param}, true)"); + } + sql += " AND (" + string.Join(" OR ", locationConditions) + ")"; + } + + // JobLevel: search in tags array (case-insensitive contains) + if (!string.IsNullOrWhiteSpace(jobquery.JobLevel)) + { + qd = qd.WithParameter("@jobLevel", jobquery.JobLevel); + // Use EXISTS with an IN on the tags array and CONTAINS for case-insensitive matching + sql += " AND EXISTS(SELECT VALUE t FROM t IN c.tags WHERE CONTAINS(t, @jobLevel, true))"; + } + + // Date range (JobPostedTime) + if (jobquery.StartDate > DateTime.MinValue) + { + qd = qd.WithParameter("@startDate", jobquery.StartDate); + sql += " AND c.jobPostedTime >= @startDate"; + } + if (jobquery.EndDate > DateTime.MinValue) + { + qd = qd.WithParameter("@endDate", jobquery.EndDate); + sql += " AND c.jobPostedTime <= @endDate"; + } + + // final ordering / limit - optional, keep callers responsible if needed + qd = new QueryDefinition(sql); // rebuild with final SQL + // re-add parameters (QueryDefinition is immutable-like with chaining, but to keep it simple rebuild) + // Add parameters again + if (!string.IsNullOrWhiteSpace(jobquery.JobType)) qd = qd.WithParameter("@jobType", jobquery.JobType); + if (jobquery.Companies != null) + { + for (int i = 0; i < jobquery.Companies.Count; i++) qd = qd.WithParameter($"@company{i}", jobquery.Companies[i]); + } + if (jobquery.Locations != null) + { + for (int i = 0; i < jobquery.Locations.Count; i++) qd = qd.WithParameter($"@location{i}", jobquery.Locations[i]); + } + if (!string.IsNullOrWhiteSpace(jobquery.JobLevel)) qd = qd.WithParameter("@jobLevel", jobquery.JobLevel); + if (jobquery.StartDate > DateTime.MinValue) qd = qd.WithParameter("@startDate", jobquery.StartDate); + if (jobquery.EndDate > DateTime.MinValue) qd = qd.WithParameter("@endDate", jobquery.EndDate); + + logger.LogInformation($"Constructed job query: {sql}"); + + return await QueryJobsAsync(qd); + } } } diff --git a/src/Common/Repositories/ProblemRepository.cs b/src/Common/Repositories/ProblemRepository.cs index 8ca5d29..1d5fa79 100644 --- a/src/Common/Repositories/ProblemRepository.cs +++ b/src/Common/Repositories/ProblemRepository.cs @@ -1,4 +1,5 @@ -using Common.Enums; +using Common.DatabaseModels; +using Common.Enums; using Common.Factories; using Common.Models; using Microsoft.Azure.Cosmos; diff --git a/src/PetProjectAzFunctions/.gitignore b/src/PetProjectAzFunctions/.gitignore new file mode 100644 index 0000000..ff5b00c --- /dev/null +++ b/src/PetProjectAzFunctions/.gitignore @@ -0,0 +1,264 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# Azure Functions localsettings file +local.settings.json + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +#*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc \ No newline at end of file diff --git a/src/PetProjectAzFunctions/Dockerfile b/src/PetProjectAzFunctions/Dockerfile new file mode 100644 index 0000000..61200dd --- /dev/null +++ b/src/PetProjectAzFunctions/Dockerfile @@ -0,0 +1,29 @@ +# See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. + +# This stage is used when running from VS in fast mode (Default for Debug configuration) +FROM mcr.microsoft.com/azure-functions/dotnet-isolated:4-dotnet-isolated8.0 AS base +WORKDIR /home/site/wwwroot +EXPOSE 8080 + + +# This stage is used to build the service project +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["PetProjectAzFunctions/PetProjectAzFunctions.csproj", "PetProjectAzFunctions/"] +RUN dotnet restore "./PetProjectAzFunctions/PetProjectAzFunctions.csproj" +COPY . . +WORKDIR "/src/PetProjectAzFunctions" +RUN dotnet build "./PetProjectAzFunctions.csproj" -c $BUILD_CONFIGURATION -o /app/build + +# This stage is used to publish the service project to be copied to the final stage +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "./PetProjectAzFunctions.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false + +# This stage is used in production or when running from VS in regular mode (Default when not using the Debug configuration) +FROM base AS final +WORKDIR /home/site/wwwroot +COPY --from=publish /app/publish . +ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ + AzureFunctionsJobHost__Logging__Console__IsEnabled=true \ No newline at end of file diff --git a/src/PetProjectAzFunctions/JobOpeningsSyncFunction.cs b/src/PetProjectAzFunctions/JobOpeningsSyncFunction.cs new file mode 100644 index 0000000..a87b8db --- /dev/null +++ b/src/PetProjectAzFunctions/JobOpeningsSyncFunction.cs @@ -0,0 +1,67 @@ +using System; +using Common.Managers; +using Common.Repositories; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace PetProjectAzFunctions +{ + public class JobOpeningsSyncFunction + { + private readonly ILogger _logger; + + private readonly JobScrapperSettingsRepository _jobScrapperSettingsRepository; + + private readonly IServiceProvider _serviceProvider; + + public JobOpeningsSyncFunction(ILoggerFactory loggerFactory, + JobScrapperSettingsRepository jobScrapperSettingsRepository, + IServiceProvider serviceProvider) + { + _logger = loggerFactory.CreateLogger(); + _jobScrapperSettingsRepository = jobScrapperSettingsRepository; + _serviceProvider = serviceProvider; + } + + [Function("JobOpeningsSyncFunction")] + public async Task Run([TimerTrigger("%CronPeriod%")] TimerInfo myTimer) + { + _logger.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}"); + var scrapperSettings = await _jobScrapperSettingsRepository.GetAllSettings(); + var currentTime = DateTime.UtcNow; + await Parallel.ForEachAsync(scrapperSettings, async (setting, ct) => + { + try + { + if (setting.enabled) + { + if(setting.lastRunTime.AddMinutes(setting.runIntervalInMinutes) >= currentTime.AddMinutes(-1)) + { + using var scope = _serviceProvider.CreateScope(); + var scrapperInstance = scope.ServiceProvider.GetRequiredService(); + scrapperInstance.ConfigureSettings(setting); + await scrapperInstance.RunAsync(); + setting.lastRunTime = currentTime; + await _jobScrapperSettingsRepository.UpdateSettingsAsync(setting.id, setting); + } + else + { + _logger.LogInformation($"Scrapper setting {setting.id} was run at {setting.lastRunTime}, next run schedule has not yet come. Skipping this run."); + } + } + else + { + _logger.LogInformation($"Scrapper setting {setting.id} is disabled. Skipping."); + return; + } + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error processing scrapper settings: {setting}"); + } + }); + } + } +} diff --git a/src/PetProjectAzFunctions/PetProjectAzFunctions.csproj b/src/PetProjectAzFunctions/PetProjectAzFunctions.csproj new file mode 100644 index 0000000..873e3b4 --- /dev/null +++ b/src/PetProjectAzFunctions/PetProjectAzFunctions.csproj @@ -0,0 +1,37 @@ + + + net8.0 + v4 + Exe + enable + enable + /home/site/wwwroot + Linux + + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + Never + + + + + + \ No newline at end of file diff --git a/src/PetProjectAzFunctions/Program.cs b/src/PetProjectAzFunctions/Program.cs new file mode 100644 index 0000000..7de0b9e --- /dev/null +++ b/src/PetProjectAzFunctions/Program.cs @@ -0,0 +1,53 @@ +using Common.Constants; +using Common.Engines; +using Common.Factories; +using Common.Managers; +using Common.Repositories; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Functions.Worker.Builder; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using System.Data; + +public class Program +{ + public static void Main(string[] args) + { + var builder = FunctionsApplication.CreateBuilder(args); + + builder.ConfigureFunctionsWebApplication(); + ConfigureServices(builder); + builder.Build().Run(); + } + + private static void ConfigureServices(FunctionsApplicationBuilder builder) + { + var services = builder.Services; + // Register your services here + services.AddLogging(); + services.AddHttpClient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + var config = builder.Configuration; + + #region Register Cosmos related services + services.AddSingleton(s => + { + var cosmosDbUri = config[ConfigurationConstants.CosmosDBUriKey]; + var cosmosDbAccountKey = config[ConfigurationConstants.CosmosDBAccountKey]; + if (string.IsNullOrEmpty(cosmosDbUri) || string.IsNullOrEmpty(cosmosDbAccountKey)) + { + throw new DataException("Cosmos DB configuration is missing or invalid."); + } + return new CosmosClient(cosmosDbUri, cosmosDbAccountKey); + }); + + services.AddTransient(); + #endregion + + } +} diff --git a/src/PetProjectAzFunctions/Properties/launchSettings.json b/src/PetProjectAzFunctions/Properties/launchSettings.json new file mode 100644 index 0000000..6a6168e --- /dev/null +++ b/src/PetProjectAzFunctions/Properties/launchSettings.json @@ -0,0 +1,15 @@ +{ + "profiles": { + "PetProjectAzFunctions": { + "commandName": "Project", + "commandLineArgs": "--port 7149" + }, + "Container (Dockerfile)": { + "commandName": "Docker", + "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}", + "containerRunArguments": "--init", + "httpPort": 31027, + "useSSL": false + } + } +} \ No newline at end of file diff --git a/src/PetProjectAzFunctions/Properties/serviceDependencies.json b/src/PetProjectAzFunctions/Properties/serviceDependencies.json new file mode 100644 index 0000000..df4dcc9 --- /dev/null +++ b/src/PetProjectAzFunctions/Properties/serviceDependencies.json @@ -0,0 +1,11 @@ +{ + "dependencies": { + "appInsights1": { + "type": "appInsights" + }, + "storage1": { + "type": "storage", + "connectionId": "AzureWebJobsStorage" + } + } +} \ No newline at end of file diff --git a/src/PetProjectAzFunctions/host.json b/src/PetProjectAzFunctions/host.json new file mode 100644 index 0000000..ee5cf5f --- /dev/null +++ b/src/PetProjectAzFunctions/host.json @@ -0,0 +1,12 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true + } + } +} \ No newline at end of file diff --git a/src/PetProjectAzFunctions/readme.md b/src/PetProjectAzFunctions/readme.md new file mode 100644 index 0000000..0b247b5 --- /dev/null +++ b/src/PetProjectAzFunctions/readme.md @@ -0,0 +1,11 @@ +# TimerTrigger - C# + +The `TimerTrigger` makes it incredibly easy to have your functions executed on a schedule. This sample demonstrates a simple use case of calling your function every 5 minutes. + +## How it works + +For a `TimerTrigger` to work, you provide a schedule in the form of a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression)(See the link for full details). A cron expression is a string with 6 separate expressions which represent a given schedule via patterns. The pattern we use to represent every 5 minutes is `0 */5 * * * *`. This, in plain text, means: "When seconds is equal to 0, minutes is divisible by 5, for any hour, day of the month, month, day of the week, or year". + +## Learn more + + Documentation \ No newline at end of file diff --git a/src/Synchronizer/ProblemsProcessor.cs b/src/Synchronizer/ProblemsProcessor.cs index db86026..a1a66d8 100644 --- a/src/Synchronizer/ProblemsProcessor.cs +++ b/src/Synchronizer/ProblemsProcessor.cs @@ -1,5 +1,6 @@ using Microsoft.Azure.Cosmos; using Common.Models; +using Common.DatabaseModels; namespace Synchronizer; diff --git a/src/lcw.sln b/src/lcw.sln index ec43a01..424b481 100644 --- a/src/lcw.sln +++ b/src/lcw.sln @@ -9,10 +9,9 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Backend", "Backend\Backend. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Synchronizer", "Synchronizer\Synchronizer.csproj", "{BF0FF8B1-3D65-459E-8CA1-A7C0ED4F97B9}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PetProjectAzFunctions", "PetProjectAzFunctions\PetProjectAzFunctions.csproj", "{31C50D63-3018-4679-92FD-F080D47A32D0}" +EndProject Global - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Debug|x64 = Debug|x64 @@ -58,5 +57,20 @@ Global {BF0FF8B1-3D65-459E-8CA1-A7C0ED4F97B9}.Release|x64.Build.0 = Release|Any CPU {BF0FF8B1-3D65-459E-8CA1-A7C0ED4F97B9}.Release|x86.ActiveCfg = Release|Any CPU {BF0FF8B1-3D65-459E-8CA1-A7C0ED4F97B9}.Release|x86.Build.0 = Release|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Debug|x64.ActiveCfg = Debug|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Debug|x64.Build.0 = Debug|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Debug|x86.ActiveCfg = Debug|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Debug|x86.Build.0 = Debug|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Release|Any CPU.Build.0 = Release|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Release|x64.ActiveCfg = Release|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Release|x64.Build.0 = Release|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Release|x86.ActiveCfg = Release|Any CPU + {31C50D63-3018-4679-92FD-F080D47A32D0}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE EndGlobalSection EndGlobal