Merge pull request #14 from devidnyk/users/devi/jobs-backend

Updated JobDetails, added more query, and adeed problems metadata

Author: devidnyk

src/Backend/Backend.csproj

@@ -17,6 +17,7 @@
     <PackageReference Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.23.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.6" />
+    <PackageReference Include="System.Runtime.Caching" Version="9.0.10" />
   </ItemGroup>
 
   <ItemGroup>

src/Backend/Controllers/JobSearchController.cs

@@ -14,47 +14,37 @@ namespace Backend.Controllers
     [Route("api/jobs")]
     public class JobSearchController : ControllerBase
     {
-        private readonly JobsRepository jobsRepository;
+        private readonly JobDataProvider jobDataProvider;
         private readonly ILogger<JobSearchController> logger;
-        public JobSearchController(JobsRepository jobsRepository, ILogger<JobSearchController> logger)
+        public JobSearchController(JobDataProvider jobDataProvider, ILogger<JobSearchController> logger)
         {
             this.logger = logger;
-            this.jobsRepository = jobsRepository;
+            this.jobDataProvider = jobDataProvider;
         }
 
         [HttpPost]
         [Route("search")]
         public async Task<ActionResult<List<ScrappedJob>>> SearchJobs([FromBody] JobQuery jobquery)
         {
-            return Ok(await jobsRepository.GetJobsFromQuery(jobquery));
+            return await this.jobDataProvider.GetJobsAsync(jobquery);
         }
 
         [HttpGet]
         [Route("latest")]
         public async Task<ActionResult<string>> GetLatestJobsFromDb(
             [FromQuery] string location = "India",
-            [FromQuery] string level = "Mid")
+            [FromQuery] string level = "Mid",
+            [FromQuery] int days = 3)
         {
-            return Content(JobListView.RenderScrappedJobsHtml(await this.jobsRepository.GetJobsEasyQueryAsync(location, level)), "text/html");
+            var jobList = await this.jobDataProvider.GetJobsAsync(location, days, level);
+            return Content(JobListView.RenderScrappedJobsHtml(jobList), "text/html");
         }
 
         [HttpGet]
         [Route("lastOneDay")]
         public async Task<ActionResult<string>> GetLastOneDayJobsFromDb()
         {
-            return Ok(await this.jobsRepository.GetAllJobsInLastOneDay());
-        }
-
-        [HttpGet]
-        [Route("profile/{id}")]
-        public async Task<ActionResult<string>> GetJobById(string id)
-        {
-            var job = await this.jobsRepository.GetJobByIdAsync(id);
-            if (job != null)
-            {
-                return Ok(job);
-            }
-            return Ok("Not found.");
+            return Ok(await this.jobDataProvider.GetAllJobsAsync(1));
         }
     }
 }

src/Backend/Controllers/ProblemsController.cs

@@ -44,6 +44,13 @@ public async Task<ActionResult<IEnumerable<Problem>>> GetProblems(
             return Ok(filteredProblems);
         }
 
+        [HttpGet]
+        [Route("problems-metadata")]
+        public async Task<ActionResult<string>> GetProblemsMetadata()
+        {
+            return Ok(await dataProvider.GetProblemsMetadataAsync());
+        }
+
         [HttpGet]
         [Route("problems/{id}")]
         public async Task<ActionResult<Problem>> GetProblems(string id)

src/Backend/Operations/DataProvider.cs

@@ -10,12 +10,37 @@ public class DataProvider
     {
         private ICache _problemCache;
         private ILogger<DataProvider> _logger;
+        private List<string> companyTags = new List<string>();
+        private DateTime lastMetadataFetchTime = DateTime.MinValue;
         public DataProvider([FromKeyedServices(CacheConstants.ProblemCacheKey)] ICache problemCache, ILogger<DataProvider> logger)
         {
             _problemCache = problemCache;
             _logger = logger;
         }
 
+        public async Task<List<string>> GetProblemsMetadataAsync()
+        {
+            if (companyTags == null || companyTags.Count == 0 || lastMetadataFetchTime < DateTime.UtcNow.AddDays(14))
+            {
+                var allProblems =  await GetAllProblemsAsync();
+                var companySet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+                foreach (var problem in allProblems.Values)
+                {
+                    if (problem.companies != null)
+                    {
+                        foreach (var company in problem.companies)
+                        {
+                            companySet.Add(company.Key);
+                        }
+                    }
+                }
+                lastMetadataFetchTime = DateTime.UtcNow;
+                companyTags = companySet?.ToList() ?? new List<string>();
+            }
+
+            return companyTags;
+        }
+
         public async Task<List<Problem>> GetProblemsAsync(IFilter<Problem>? filter = null)
         {
             var allProblems = await GetAllProblemsAsync();

src/Backend/Operations/JobDataProvider.cs

@@ -0,0 +1,78 @@
+namespace Backend.Operations
+{
+    using System.ComponentModel;
+    using Common.Cache;
+    using Common.DatabaseModels;
+    using Common.Queries;
+    using Common.Repositories;
+    
+    public class JobDataProvider
+    {
+        private readonly ILogger<JobDataProvider> _logger;
+        private readonly JobsRepository _jobsRepository;
+
+        private TimeSpan cacheDuration = TimeSpan.FromMinutes(15);
+        private Memca cache;
+
+        public JobDataProvider(ILogger<JobDataProvider> logger, JobsRepository jobsRepository)
+        {
+            _logger = logger;
+            _jobsRepository = jobsRepository;
+            this.cache = new Memca(this.cacheDuration);
+        }
+
+        public async Task<List<ScrappedJob>> GetJobsAsync(string jobLocation, int lookbackdays, string level)
+        {
+            var filteredJobs = await GetRecentJobsAsync(jobLocation, lookbackdays);
+
+            if (!string.IsNullOrEmpty(level))
+            {
+                filteredJobs = filteredJobs.Where(j => j.jobType.ToLower().Contains(level.ToLower())).ToList();
+            }
+
+            return filteredJobs;
+        }
+
+        public async Task<List<ScrappedJob>> GetJobsAsync(JobQuery jobquery)
+        {
+            return await this._jobsRepository.GetJobsFromQuery(jobquery);
+        }
+        
+        public async Task<List<ScrappedJob>> GetAllJobsAsync(int lookbackdays = 1)
+        {
+            var allJobs = await GetAllLatestJobsAsync();
+            allJobs = allJobs.Where(j => j.scrappedTime >= DateTime.UtcNow.AddDays(-lookbackdays)).ToList();
+            return allJobs;
+        }
+
+        private async Task<List<ScrappedJob>> GetRecentJobsAsync(string location = "india", int lookbackdays = 3)
+        {
+            string cacheKey = $"jobs_{location}_{lookbackdays}";
+            if (this.cache.Get<List<ScrappedJob>>(cacheKey) is List<ScrappedJob> cachedJobs)
+            {
+                _logger.LogInformation($"Cache hit for key: {cacheKey}");
+                return cachedJobs;
+            }
+
+            _logger.LogInformation($"Cache miss for key: {cacheKey}. Fetching from database.");
+            var jobs = await _jobsRepository.GetJobsAsync(location, lookbackdays);
+            this.cache.Set(cacheKey, jobs, this.cacheDuration);
+            return jobs;
+        }
+
+        private async Task<List<ScrappedJob>> GetAllLatestJobsAsync()
+        {
+            string cacheKey = $"all_jobs_latest";
+            if (this.cache.Get<List<ScrappedJob>>(cacheKey) is List<ScrappedJob> cachedJobs)
+            {
+                _logger.LogInformation($"Cache hit for key: {cacheKey}");
+                return cachedJobs;
+            }
+
+            _logger.LogInformation($"Cache miss for key: {cacheKey}. Fetching from database.");
+            var jobs = await _jobsRepository.GetAllLatestJobsAsync();
+            this.cache.Set(cacheKey, jobs, this.cacheDuration);
+            return jobs;
+        }
+    }
+}

src/Backend/Program.cs

@@ -103,6 +103,7 @@ public static void Main(string[] args)
         services.AddSingleton<JobsRepository>();
         services.AddSingleton<JobScrapperSettingsManager>();
         services.AddSingleton<ScrapperRunner>();
+        services.AddSingleton<JobDataProvider>();
 
         var app = builder.Build();
         ILogger logger = app.Logger;

src/Common/Cache/Memca.cs

@@ -0,0 +1,39 @@
+namespace Common.Cache
+{
+    using System.Runtime.Caching;
+    public class Memca
+    {
+        private MemoryCache cache;
+        private readonly TimeSpan defaultExpiryDuration;
+
+        public Memca(TimeSpan defaultExpiryDuration)
+        {
+            this.cache = MemoryCache.Default;
+            this.defaultExpiryDuration = defaultExpiryDuration;
+        }
+
+        public T? Get<T>(string key)
+        {
+            return (T?)this.cache.Get(key);
+        }
+
+        public void Set<T>(string key, T value, TimeSpan? absoluteExpiration = null)
+        {
+            var policy = new CacheItemPolicy();
+            if (absoluteExpiration.HasValue)
+            {
+                policy.AbsoluteExpiration = DateTimeOffset.Now.Add(absoluteExpiration.Value);
+            }
+            else
+            {
+                policy.AbsoluteExpiration = DateTimeOffset.Now.Add(this.defaultExpiryDuration);
+            }
+            this.cache.Set(key, value, policy);
+        }
+
+        public void Remove(string key)
+        {
+            this.cache.Remove(key);
+        }
+    }    
+}

src/Common/Common.csproj

@@ -16,6 +16,7 @@
 	<PackageReference Include="Azure.AI.Agents.Persistent" Version="1.1.0" />
 	<PackageReference Include="Azure.AI.Inference" Version="1.0.0-beta.5" />
 	<PackageReference Include="Azure.AI.Projects" Version="1.0.0" />
+	<PackageReference Include="System.Runtime.Caching" Version="9.0.10" />
   </ItemGroup>
 
 </Project>

src/Common/Managers/JobScrapper.cs

@@ -40,6 +40,19 @@ public async Task<string> RunAsync()
                 return "No search results, processing skipped.";
             }
 
+            var existingJobIds = await this.jobsRepository.GetJobIdsInLastNDaysAsync(1);
+            var hashSetExistingJobIds = new HashSet<string>(existingJobIds, StringComparer.OrdinalIgnoreCase);
+            
+            this.logger.LogInformation($"Search Results count: {searchResults.Count} JobIds in last N(1) days = {hashSetExistingJobIds.Count}");
+            searchResults = searchResults.Where(j => !hashSetExistingJobIds.Contains(j.id)).ToList();
+            this.logger.LogInformation($"Filtered Search Results count: {searchResults.Count}");
+
+            if (searchResults.Count == 0)
+            {
+                this.logger.LogInformation($"All jobs are duplicates. Nothing to process. Query settings: {this.settings}");
+                return "All jobs are duplicates, processing skipped.";
+            }
+
             var mp = new Dictionary<string, ScrappedJob>(StringComparer.OrdinalIgnoreCase);
             foreach (var job in searchResults)
             {
@@ -48,15 +61,18 @@ public async Task<string> RunAsync()
                     mp[job.id] = job;
                 }
             }
-            
-            // TODO: Filter duplicates by fetching the latest jobs from DB in last 1d.
-            
+
             var levels = await this.aiEngine.GetJobLevelAsync(searchResults);
             foreach (var level in levels)
             {
                 if (mp.ContainsKey(level.Key))
                 {
                     mp[level.Key].tags.AddRange(level.Value.Split("-"));
+                    mp[level.Key].jobType = level.Value.Split("-").FirstOrDefault() ?? mp[level.Key].jobType;
+                    if (level.Value.Split("-").Length > 1)
+                    {
+                        mp[level.Key].location = level.Value.Split("-")[1];
+                    }
                 }
                 else
                 {

src/Common/Repositories/JobsRepository.cs

@@ -22,7 +22,7 @@ public JobsRepository(ICosmosContainerFactory cosmosContainerFactory,
 
         public async Task<List<ScrappedJob>> GetAllLatestJobsAsync()
         {
-            var query = "SELECT * FROM c ORDER BY c.scrappedTime DESC OFFSET 0 LIMIT 100";
+            var query = "SELECT * FROM c ORDER BY c.scrappedTime DESC OFFSET 0 LIMIT 2000";
             return await QueryJobsAsync(query);
         }
 
@@ -32,26 +32,6 @@ public async Task<List<ScrappedJob>> GetAllJobsInLastOneDay()
             return await QueryJobsAsync(query);
         }
 
-        public async Task<ScrappedJob> GetJobByIdAsync(string id)
-        {
-            try
-            {
-                // TODO: NOT working as expected
-                var response = await this.jobsContainer.ReadItemAsync<ScrappedJob>(id, new PartitionKey(id));
-                return response.Resource;
-            }
-            catch (CosmosException cosmosEx) when (cosmosEx.StatusCode == System.Net.HttpStatusCode.NotFound)
-            {
-                this.logger.LogWarning($"Job: {id} not found in container.");
-                return null;
-            }
-            catch (Exception ex)
-            {
-                this.logger.LogError($"Failed to retrieve job: {id} from container. Ex: {ex}");
-                return null;
-            }
-        }
-
         /// <summary>
         /// Create the item only if it does not already exist using a single DB call.
         /// Returns true if the item was created, false if it already existed.
@@ -95,6 +75,32 @@ public async Task<List<ScrappedJob>> GetJobsEasyQueryAsync(string location, stri
             return res;
         }
 
+        public async Task<List<string>> GetJobIdsInLastNDaysAsync(int lookbackdays)
+        {
+            var cutoffDate = DateTime.UtcNow.AddDays(-lookbackdays);
+            var query = "SELECT c.id FROM c WHERE c.scrappedTime >= @cutoffDate";
+            var queryDefinition = new QueryDefinition(query).WithParameter("@cutoffDate", cutoffDate);
+            var queryResultSetIterator = jobsContainer.GetItemQueryIterator<ScrappedJob>(queryDefinition);
+            List<string> results = new List<string>();
+            while (queryResultSetIterator.HasMoreResults)
+            {
+                var response = await queryResultSetIterator.ReadNextAsync();
+                results.AddRange(response.Select(j => j.id));
+            }
+            this.logger.LogInformation($"Retrieved {results.Count} job IDs from Cosmos DB in last {lookbackdays} days.");
+            return results;
+        }
+
+        public async Task<List<ScrappedJob>> GetJobsAsync(string location, int lookbackdays)
+        {
+            var cutoffDate = DateTime.UtcNow.AddDays(-lookbackdays);
+            var query = "SELECT * FROM c WHERE (LOWER(c.location) = @location OR LOWER(c.location) = @unknown) AND c.scrappedTime >= @cutoffDate ORDER BY c.scrappedTime DESC";
+            var queryDefinition = new QueryDefinition(query)
+                .WithParameter("@location", location.ToLower())
+                .WithParameter("@unknown", "unknown")
+                .WithParameter("@cutoffDate", cutoffDate.ToString("yyyy-MM-ddTHH:mm:ss"));
+            return await QueryJobsAsync(queryDefinition);
+        }
 
         private async Task<List<ScrappedJob>> QueryJobsAsync(string query)
         {