Optimize the getProducts performance as much as possible with making sure it returns the…
- User
Optimize the getProducts performance as much as possible with making sure it returns the same result as it is now... also improve the stability of it and of filtering and searching... Also remove console.log... Revise your solution and make sure it is the best... I'm hosting my express app in 1gb ram vps... productController.ts: import { Request, Response } from 'express'; import {query} from "../config/database"; import {brandsCache, productCache} from "../cache"; import { Brand, BrandsCacheData, CacheData, Color, CreateProductInput, Image, ImageInput, Product, Size, UpdateProductInput } from "product"; import {addProductToCache, deleteProductFromCache, updateProductInCache} from "../cacheHelper";
const fetchCategoryDetails = async (subcategoryIds: number[]) => { const categoriesResult = await query(
SELECT sc.id AS subcategory_id, sc.name AS subcategory_name, c.id AS main_category_id, c.name AS main_category_name FROM subcategories sc JOIN categories c ON sc.category_id = c.id WHERE sc.id = ANY($1::int[]), [subcategoryIds] );return categoriesResult.rows.map((row: any) => ({ id: row.subcategory_id, name: row.subcategory_name, main_category_id: row.main_category_id, main_category_name: row.main_category_name }));}; export const loadAndCacheAllProducts = async (): Promise => { const q =
SELECT p.id, p.name, p.description, p.brand_id, p.has_color, p.has_size, p.product_type, p.price, p.qty, p.sold, p.created_at, p.updated_at, sc.id AS subcategory_id, sc.name AS subcategory_name, c.id AS main_category_id, c.name AS main_category_name, s.id AS size_id, s.name AS size_name, s.price AS size_price, s.qty AS size_qty, s.sold AS size_sold, cl.id AS color_id, cl.name AS color_name, cl.price AS color_price, cl.qty AS color_qty, cl.sold AS color_sold, i.id AS image_id, i.url AS image_url, i.color_id AS image_color_id, i.size_id AS image_size_id FROM products p LEFT JOIN product_categories pc ON p.id = pc.product_id LEFT JOIN subcategories sc ON pc.category_id = sc.id LEFT JOIN categories c ON sc.category_id = c.id LEFT JOIN colors cl ON p.id = cl.product_id LEFT JOIN sizes s ON p.id = s.product_id LEFT JOIN images i ON (p.id = i.product_id AND (cl.id = i.color_id OR i.color_id IS NULL) AND (s.id = i.size_id OR i.size_id IS NULL));try { const result = await query(q, []); const productsMap: Record<number, Product> = {}; // Temporary map to hold sizes that need to be assigned to colors const sizesMap: Record<number, Size> = {}; // First pass: Initialize products and collect sizes result.rows.forEach((row: any) => { const productId = Number(row.id); // Initialize the product in the map if it doesn't exist if (!productsMap[productId]) { productsMap[productId] = { id: productId, name: row.name, description: row.description, brand_id: Number(row.brand_id), has_color: Boolean(row.has_color), has_size: Boolean(row.has_size), product_type: row.product_type, price: Number(row.price), qty: Number(row.qty), sold: Number(row.sold), created_at: new Date(row.created_at).toISOString(), updated_at: new Date(row.updated_at).toISOString(), categories: [], colors: [], sizes: [], // Will be populated only if has_color is false images: [] }; } const product = productsMap[productId]; // Add category if present and not already added if ( row.subcategory_id && row.subcategory_name && row.main_category_id && row.main_category_name ) { const categoryId = Number(row.subcategory_id); const mainCategoryId = Number(row.main_category_id); const existingCategory = product.categories.find(cat => cat.id === categoryId); if (!existingCategory) { product.categories.push({ id: categoryId, name: row.subcategory_name, main_category_id: mainCategoryId, main_category_name: row.main_category_name, }); } } // Add color if present and not already added if (row.color_id && row.color_name && row.has_size !== undefined) { const colorId = Number(row.color_id); let color = product.colors.find(color => color.id === colorId); if (!color) { color = { id: colorId, name: row.color_name, price: Number(row.color_price), qty: Number(row.color_qty), sold: Number(row.color_sold), has_size: Boolean(row.has_size), sizes: [], // Will be populated if has_size is true images: [] }; product.colors.push(color); } } // Collect sizes to assign later based on color association if (row.size_id && row.size_name) { const sizeId = Number(row.size_id); const colorId = row.color_id ? Number(row.color_id) : null; // If the size is associated with a color, store it for later assignment if (colorId && product.has_color) { sizesMap[sizeId] = { id: sizeId, name: row.size_name, price: Number(row.size_price), qty: Number(row.size_qty), sold: Number(row.size_sold), color_id: colorId, images: [] }; } else if (!product.has_color && product.has_size) { // If the product has sizes but no colors, assign size directly if (!product.sizes.find(s => s.id === sizeId)) { product.sizes.push({ id: sizeId, name: row.size_name, price: Number(row.size_price), qty: Number(row.size_qty), sold: Number(row.size_sold), color_id: null, images: [] }); } } } // Collect images to assign if (row.image_id && row.image_url) { const imageId = Number(row.image_id); // Avoid duplicate images if (!product.images.find(img => img.id === imageId)) { product.images.push({ id: imageId, url: row.image_url, color_id: row.image_color_id ? Number(row.image_color_id) : null, size_id: row.image_size_id ? Number(row.image_size_id) : null }); } // Associate image with color if applicable if (row.image_color_id) { const colorId = Number(row.image_color_id); const color = product.colors.find(c => c.id === colorId); if (color && !color.images.includes(row.image_url)) { color.images.push(row.image_url); } } // Associate image with size if applicable and size is within a color if (row.image_size_id) { const sizeId = Number(row.image_size_id); const size = sizesMap[sizeId]; if (size) { if (!size.images.includes(row.image_url)) { size.images.push(row.image_url); } } else { // If the product does not have colors, associate with product's sizes const standaloneSize = product.sizes.find(s => s.id === Number(row.image_size_id)); if (standaloneSize && !standaloneSize.images.includes(row.image_url)) { standaloneSize.images.push(row.image_url); } } } } }); // Second pass: Assign sizes to their respective colors Object.values(productsMap).forEach(product => { if (product.has_color) { // Initialize all product.sizes as empty product.sizes = []; // Iterate over collected sizes and assign to colors Object.values(sizesMap).forEach(size => { if (size.color_id === null) return; // Skip sizes without color_id const color = product.colors.find(c => c.id === size.color_id); if (color) { color.sizes.push({ id: size.id, name: size.name, price: size.price, qty: size.qty, sold: size.sold, color_id: size.color_id, images: size.images }); } }); // Update has_size for each color based on whether it has sizes product.colors.forEach(color => { color.has_size = color.sizes.length > 0; }); } // If has_color is false, sizes are already assigned directly to product.sizes }); const allProducts = Object.values(productsMap); // Calculate min and max prices across all products let minPrice = Infinity; let maxPrice = -Infinity; allProducts.forEach(product => { // Product price if (product.price < minPrice) minPrice = product.price; if (product.price > maxPrice) maxPrice = product.price; // Prices from colors and their sizes product.colors.forEach(color => { if (color.price < minPrice) minPrice = color.price; if (color.price > maxPrice) maxPrice = color.price; color.sizes.forEach(size => { if (size.price < minPrice) minPrice = size.price; if (size.price > maxPrice) maxPrice = size.price; }); }); // Prices from standalone sizes product.sizes.forEach(size => { if (size.price < minPrice) minPrice = size.price; if (size.price > maxPrice) maxPrice = size.price; }); }); // Fallback in case no prices are present if (minPrice === Infinity) minPrice = 0; if (maxPrice === -Infinity) maxPrice = 0; // Cache the data with a consistent key const cacheData: CacheData = { allProducts, minPrice, maxPrice }; productCache.set('allProducts', cacheData); console.log(`Cached ${allProducts.length} products with minPrice: ${minPrice} and maxPrice: ${maxPrice}.`); return cacheData; } catch (error) { console.error('Error loading products from DB:', error); throw error; // Propagate error to be handled by caller }};
/**
-
Fetches all brands from the database and caches them.
-
@returns {Promise} A promise that resolves to the cached brand data. */ const loadAndCacheAllBrands = async (): Promise => { const q =
SELECT s.*, c.name as category_name FROM subcategories s JOIN categories c ON s.category_id = c.id AND c.is_brand ORDER BY c.name, s.name;try { const result = await query(q, []); const allBrands: Brand[] = result.rows.map((row: any) => ({ id: Number(row.id), name: row.name, category_id: Number(row.category_id), category_name: row.category_name, }));
// Cache the brands const brandsCacheData: BrandsCacheData = { allBrands }; brandsCache.set('allBrands', brandsCacheData); console.log(`Cached ${allBrands.length} brands successfully.`); return brandsCacheData;} catch (error) { console.error('Error loading brands from DB:', error); throw error; // Propagate error to be handled by caller } };
/**
-
Controller to handle fetching and responding with filtered products.
-
Caches all products and brands on first request and performs in-memory filtering thereafter.
-
@param {Request} req - Express request object containing query parameters.
-
@param {Response} res - Express response object to send JSON data. */ export const getProducts = async (req: Request, res: Response): Promise => { try { // Retrieve cached products let cacheData: CacheData | undefined = productCache.get('cacheData');
if (!cacheData) { // Cache miss: Load from DB and cache cacheData = await loadAndCacheAllProducts(); console.log('Loaded products from database and cached them.'); } else { console.log('Loaded products from cache.'); } const { allProducts, minPrice, maxPrice } = cacheData; // Log total products before any filtering console.log(`Total products before filtering: ${allProducts.length}`); // Extract and sanitize query parameters let { search = '', c, sc, b, minPriceA = '0', maxPriceA = '', availability = 'all', sortBy = '', page = '1', itemsPerPage = '15', } = req.query; // Parse numerical values const pageNum = parseInt(page as string, 10) || 1; const itemsPerPageNum = parseInt(itemsPerPage as string, 10) || 15; const minPriceANum = parseFloat(minPriceA as string) || 0; let maxPriceANum = maxPriceA ? parseFloat(maxPriceA as string) : Infinity; // If maxPriceA is less than minPriceA, reset to Infinity if (!isNaN(maxPriceANum) && maxPriceANum < minPriceANum) { console.warn('Provided maxPriceA is less than minPriceA. Resetting maxPriceANum to Infinity.'); maxPriceANum = Infinity; } else if (isNaN(maxPriceANum)) { maxPriceANum = Infinity; } // Parse c, sc, b as numbers if present const cNum = c ? parseInt(c as string, 10) : undefined; const scNum = sc ? parseInt(sc as string, 10) : undefined; const bNum = b ? parseInt(b as string, 10) : undefined; // Initialize brand-related variables if 'b' is provided let allBrands: Brand[] = []; if (bNum !== undefined && !isNaN(bNum)) { // Retrieve cached brands let brandsCacheData: BrandsCacheData | undefined = brandsCache.get('allBrands'); if (!brandsCacheData) { // Cache miss: Load from DB and cache brandsCacheData = await loadAndCacheAllBrands(); console.log('Loaded brands from database and cached them.'); } else { console.log('Loaded brands from cache.'); } console.log(brandsCacheData) allBrands = brandsCacheData.allBrands; // Check if 'bNum' is a valid brand ID const isValidBrand = allBrands.some(brand => brand.category_id === bNum); if (!isValidBrand) { console.warn(`Brand ID ${bNum} is invalid.`); // Return zero products since 'b' is invalid res.json({ products: [], total: 0, page: pageNum, itemsPerPage: itemsPerPageNum, totalPages: 0, hasMore: false, minPrice, maxPrice, }); return; } } // Start with all products let filteredProducts = [...allProducts]; // Apply Search Filter if (search && typeof search === 'string') { const searchLower = search.toLowerCase(); const searchWords = searchLower.split(/\s+/).filter(Boolean); filteredProducts = filteredProducts.filter(product => searchWords.every(word => product.name.toLowerCase().includes(word)) ); } // Apply Category Filter (c) if (cNum !== undefined && !isNaN(cNum)) { filteredProducts = filteredProducts.filter(product => product.categories.some(cat => cat.main_category_id === cNum) ); } // Apply Subcategory Filter (sc) if (scNum !== undefined && !isNaN(scNum)) { filteredProducts = filteredProducts.filter(product => product.brand_id === scNum || product.categories.some(cat => cat.id === scNum) ); } // Apply Brand Filter (b) if (bNum !== undefined && !isNaN(bNum)) { console.log(`Applying Brand Filter: brand_id === ${bNum}`); const preFilterCount = filteredProducts.length; allBrands = allBrands.filter(brand => brand.category_id === bNum) filteredProducts = filteredProducts.filter(product => allBrands.some(brand => brand.id === product.brand_id) ); console.log(`Filtered from ${preFilterCount} to ${filteredProducts.length} products after applying brand filter.`); } // Apply Price Range Filter if (minPriceANum > 0 || maxPriceANum < Infinity) { filteredProducts = filteredProducts.filter(product => { const productPrices = product.sizes.map(size => size.price); if (productPrices.length === 0) return false; const productMinPrice = Math.min(...productPrices); const productMaxPrice = Math.max(...productPrices); return productMinPrice >= minPriceANum && productMaxPrice <= maxPriceANum; }); } // Apply Availability Filter if (availability === 'in_stock') { filteredProducts = filteredProducts.filter(product => product.sizes.some(size => size.qty > 0) ); } else if (availability === 'out_of_stock') { filteredProducts = filteredProducts.filter(product => product.sizes.every(size => size.qty === 0) ); } // Log total after filtering console.log(`Total products after filtering: ${filteredProducts.length}`); // Apply Sorting if (sortBy && typeof sortBy === 'string') { switch (sortBy) { case 'Price: Low to High': filteredProducts.sort((a, b) => { const aMin = Math.min(...a.sizes.map(s => s.price)); const bMin = Math.min(...b.sizes.map(s => s.price)); return aMin - bMin; }); break; case 'Price: High to Low': filteredProducts.sort((a, b) => { const aMax = Math.max(...a.sizes.map(s => s.price)); const bMax = Math.max(...b.sizes.map(s => s.price)); return bMax - aMax; }); break; case 'Name: A to Z': filteredProducts.sort((a, b) => a.name.localeCompare(b.name)); break; case 'Name: Z to A': filteredProducts.sort((a, b) => b.name.localeCompare(a.name)); break; case 'Best Selling': filteredProducts.sort((a, b) => b.sold - a.sold); break; case 'Newest Arrivals': filteredProducts.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); break; default: // If sortBy parameter doesn't match any case, do not sort break; } } // Apply Pagination const totalFilteredProducts = filteredProducts.length; const totalPages = Math.ceil(totalFilteredProducts / itemsPerPageNum); const startIdx = (pageNum - 1) * itemsPerPageNum; const endIdx = startIdx + itemsPerPageNum; const paginatedProducts = filteredProducts.slice(startIdx, endIdx); // Construct Response const response = { products: paginatedProducts, total: totalFilteredProducts, page: pageNum, itemsPerPage: itemsPerPageNum, totalPages, hasMore: endIdx < totalFilteredProducts, minPrice, // Global minPrice across all products maxPrice, // Global maxPrice across all products }; // Send Response res.json(response);} catch (error) { console.error('Error fetching products:', error); res.status(500).json({ error: 'Internal server error' }); } finally { // Optional: Any cleanup operations can be performed here // For example, monitoring memory usage or logging request completion } };
// Rest of productController code... cache.ts: import { Request, Response } from 'express'; import {query} from "../config/database"; import {brandsCache, productCache} from "../cache"; import { Brand, BrandsCacheData, CacheData, Color, CreateProductInput, Image, ImageInput, Product, Size, UpdateProductInput } from "product"; import {addProductToCache, deleteProductFromCache, updateProductInCache} from "../cacheHelper";
const fetchCategoryDetails = async (subcategoryIds: number[]) => { const categoriesResult = await query(
SELECT sc.id AS subcategory_id, sc.name AS subcategory_name, c.id AS main_category_id, c.name AS main_category_name FROM subcategories sc JOIN categories c ON sc.category_id = c.id WHERE sc.id = ANY($1::int[]), [subcategoryIds] );return categoriesResult.rows.map((row: any) => ({ id: row.subcategory_id, name: row.subcategory_name, main_category_id: row.main_category_id, main_category_name: row.main_category_name }));}; export const loadAndCacheAllProducts = async (): Promise => { const q =
SELECT p.id, p.name, p.description, p.brand_id, p.has_color, p.has_size, p.product_type, p.price, p.qty, p.sold, p.created_at, p.updated_at, sc.id AS subcategory_id, sc.name AS subcategory_name, c.id AS main_category_id, c.name AS main_category_name, s.id AS size_id, s.name AS size_name, s.price AS size_price, s.qty AS size_qty, s.sold AS size_sold, cl.id AS color_id, cl.name AS color_name, cl.price AS color_price, cl.qty AS color_qty, cl.sold AS color_sold, i.id AS image_id, i.url AS image_url, i.color_id AS image_color_id, i.size_id AS image_size_id FROM products p LEFT JOIN product_categories pc ON p.id = pc.product_id LEFT JOIN subcategories sc ON pc.category_id = sc.id LEFT JOIN categories c ON sc.category_id = c.id LEFT JOIN colors cl ON p.id = cl.product_id LEFT JOIN sizes s ON p.id = s.product_id LEFT JOIN images i ON (p.id = i.product_id AND (cl.id = i.color_id OR i.color_id IS NULL) AND (s.id = i.size_id OR i.size_id IS NULL));try { const result = await query(q, []); const productsMap: Record<number, Product> = {}; // Temporary map to hold sizes that need to be assigned to colors const sizesMap: Record<number, Size> = {}; // First pass: Initialize products and collect sizes result.rows.forEach((row: any) => { const productId = Number(row.id); // Initialize the product in the map if it doesn't exist if (!productsMap[productId]) { productsMap[productId] = { id: productId, name: row.name, description: row.description, brand_id: Number(row.brand_id), has_color: Boolean(row.has_color), has_size: Boolean(row.has_size), product_type: row.product_type, price: Number(row.price), qty: Number(row.qty), sold: Number(row.sold), created_at: new Date(row.created_at).toISOString(), updated_at: new Date(row.updated_at).toISOString(), categories: [], colors: [], sizes: [], // Will be populated only if has_color is false images: [] }; } const product = productsMap[productId]; // Add category if present and not already added if ( row.subcategory_id && row.subcategory_name && row.main_category_id && row.main_category_name ) { const categoryId = Number(row.subcategory_id); const mainCategoryId = Number(row.main_category_id); const existingCategory = product.categories.find(cat => cat.id === categoryId); if (!existingCategory) { product.categories.push({ id: categoryId, name: row.subcategory_name, main_category_id: mainCategoryId, main_category_name: row.main_category_name, }); } } // Add color if present and not already added if (row.color_id && row.color_name && row.has_size !== undefined) { const colorId = Number(row.color_id); let color = product.colors.find(color => color.id === colorId); if (!color) { color = { id: colorId, name: row.color_name, price: Number(row.color_price), qty: Number(row.color_qty), sold: Number(row.color_sold), has_size: Boolean(row.has_size), sizes: [], // Will be populated if has_size is true images: [] }; product.colors.push(color); } } // Collect sizes to assign later based on color association if (row.size_id && row.size_name) { const sizeId = Number(row.size_id); const colorId = row.color_id ? Number(row.color_id) : null; // If the size is associated with a color, store it for later assignment if (colorId && product.has_color) { sizesMap[sizeId] = { id: sizeId, name: row.size_name, price: Number(row.size_price), qty: Number(row.size_qty), sold: Number(row.size_sold), color_id: colorId, images: [] }; } else if (!product.has_color && product.has_size) { // If the product has sizes but no colors, assign size directly if (!product.sizes.find(s => s.id === sizeId)) { product.sizes.push({ id: sizeId, name: row.size_name, price: Number(row.size_price), qty: Number(row.size_qty), sold: Number(row.size_sold), color_id: null, images: [] }); } } } // Collect images to assign if (row.image_id && row.image_url) { const imageId = Number(row.image_id); // Avoid duplicate images if (!product.images.find(img => img.id === imageId)) { product.images.push({ id: imageId, url: row.image_url, color_id: row.image_color_id ? Number(row.image_color_id) : null, size_id: row.image_size_id ? Number(row.image_size_id) : null }); } // Associate image with color if applicable if (row.image_color_id) { const colorId = Number(row.image_color_id); const color = product.colors.find(c => c.id === colorId); if (color && !color.images.includes(row.image_url)) { color.images.push(row.image_url); } } // Associate image with size if applicable and size is within a color if (row.image_size_id) { const sizeId = Number(row.image_size_id); const size = sizesMap[sizeId]; if (size) { if (!size.images.includes(row.image_url)) { size.images.push(row.image_url); } } else { // If the product does not have colors, associate with product's sizes const standaloneSize = product.sizes.find(s => s.id === Number(row.image_size_id)); if (standaloneSize && !standaloneSize.images.includes(row.image_url)) { standaloneSize.images.push(row.image_url); } } } } }); // Second pass: Assign sizes to their respective colors Object.values(productsMap).forEach(product => { if (product.has_color) { // Initialize all product.sizes as empty product.sizes = []; // Iterate over collected sizes and assign to colors Object.values(sizesMap).forEach(size => { if (size.color_id === null) return; // Skip sizes without color_id const color = product.colors.find(c => c.id === size.color_id); if (color) { color.sizes.push({ id: size.id, name: size.name, price: size.price, qty: size.qty, sold: size.sold, color_id: size.color_id, images: size.images }); } }); // Update has_size for each color based on whether it has sizes product.colors.forEach(color => { color.has_size = color.sizes.length > 0; }); } // If has_color is false, sizes are already assigned directly to product.sizes }); const allProducts = Object.values(productsMap); // Calculate min and max prices across all products let minPrice = Infinity; let maxPrice = -Infinity; allProducts.forEach(product => { // Product price if (product.price < minPrice) minPrice = product.price; if (product.price > maxPrice) maxPrice = product.price; // Prices from colors and their sizes product.colors.forEach(color => { if (color.price < minPrice) minPrice = color.price; if (color.price > maxPrice) maxPrice = color.price; color.sizes.forEach(size => { if (size.price < minPrice) minPrice = size.price; if (size.price > maxPrice) maxPrice = size.price; }); }); // Prices from standalone sizes product.sizes.forEach(size => { if (size.price < minPrice) minPrice = size.price; if (size.price > maxPrice) maxPrice = size.price; }); }); // Fallback in case no prices are present if (minPrice === Infinity) minPrice = 0; if (maxPrice === -Infinity) maxPrice = 0; // Cache the data with a consistent key const cacheData: CacheData = { allProducts, minPrice, maxPrice }; productCache.set('allProducts', cacheData); console.log(`Cached ${allProducts.length} products with minPrice: ${minPrice} and maxPrice: ${maxPrice}.`); return cacheData; } catch (error) { console.error('Error loading products from DB:', error); throw error; // Propagate error to be handled by caller }};
/**
-
Fetches all brands from the database and caches them.
-
@returns {Promise} A promise that resolves to the cached brand data. */ const loadAndCacheAllBrands = async (): Promise => { const q =
SELECT s.*, c.name as category_name FROM subcategories s JOIN categories c ON s.category_id = c.id AND c.is_brand ORDER BY c.name, s.name;try { const result = await query(q, []); const allBrands: Brand[] = result.rows.map((row: any) => ({ id: Number(row.id), name: row.name, category_id: Number(row.category_id), category_name: row.category_name, }));
// Cache the brands const brandsCacheData: BrandsCacheData = { allBrands }; brandsCache.set('allBrands', brandsCacheData); console.log(`Cached ${allBrands.length} brands successfully.`); return brandsCacheData;} catch (error) { console.error('Error loading brands from DB:', error); throw error; // Propagate error to be handled by caller } };
/**
-
Controller to handle fetching and responding with filtered products.
-
Caches all products and brands on first request and performs in-memory filtering thereafter.
-
@param {Request} req - Express request object containing query parameters.
-
@param {Response} res - Express response object to send JSON data. */ export const getProducts = async (req: Request, res: Response): Promise => { try { // Retrieve cached products let cacheData: CacheData | undefined = productCache.get('cacheData');
if (!cacheData) { // Cache miss: Load from DB and cache cacheData = await loadAndCacheAllProducts(); console.log('Loaded products from database and cached them.'); } else { console.log('Loaded products from cache.'); } const { allProducts, minPrice, maxPrice } = cacheData; // Log total products before any filtering console.log(`Total products before filtering: ${allProducts.length}`); // Extract and sanitize query parameters let { search = '', c, sc, b, minPriceA = '0', maxPriceA = '', availability = 'all', sortBy = '', page = '1', itemsPerPage = '15', } = req.query; // Parse numerical values const pageNum = parseInt(page as string, 10) || 1; const itemsPerPageNum = parseInt(itemsPerPage as string, 10) || 15; const minPriceANum = parseFloat(minPriceA as string) || 0; let maxPriceANum = maxPriceA ? parseFloat(maxPriceA as string) : Infinity; // If maxPriceA is less than minPriceA, reset to Infinity if (!isNaN(maxPriceANum) && maxPriceANum < minPriceANum) { console.warn('Provided maxPriceA is less than minPriceA. Resetting maxPriceANum to Infinity.'); maxPriceANum = Infinity; } else if (isNaN(maxPriceANum)) { maxPriceANum = Infinity; } // Parse c, sc, b as numbers if present const cNum = c ? parseInt(c as string, 10) : undefined; const scNum = sc ? parseInt(sc as string, 10) : undefined; const bNum = b ? parseInt(b as string, 10) : undefined; // Initialize brand-related variables if 'b' is provided let allBrands: Brand[] = []; if (bNum !== undefined && !isNaN(bNum)) { // Retrieve cached brands let brandsCacheData: BrandsCacheData | undefined = brandsCache.get('allBrands'); if (!brandsCacheData) { // Cache miss: Load from DB and cache brandsCacheData = await loadAndCacheAllBrands(); console.log('Loaded brands from database and cached them.'); } else { console.log('Loaded brands from cache.'); } console.log(brandsCacheData) allBrands = brandsCacheData.allBrands; // Check if 'bNum' is a valid brand ID const isValidBrand = allBrands.some(brand => brand.category_id === bNum); if (!isValidBrand) { console.warn(`Brand ID ${bNum} is invalid.`); // Return zero products since 'b' is invalid res.json({ products: [], total: 0, page: pageNum, itemsPerPage: itemsPerPageNum, totalPages: 0, hasMore: false, minPrice, maxPrice, }); return; } } // Start with all products let filteredProducts = [...allProducts]; // Apply Search Filter if (search && typeof search === 'string') { const searchLower = search.toLowerCase(); const searchWords = searchLower.split(/\s+/).filter(Boolean); filteredProducts = filteredProducts.filter(product => searchWords.every(word => product.name.toLowerCase().includes(word)) ); } // Apply Category Filter (c) if (cNum !== undefined && !isNaN(cNum)) { filteredProducts = filteredProducts.filter(product => product.categories.some(cat => cat.main_category_id === cNum) ); } // Apply Subcategory Filter (sc) if (scNum !== undefined && !isNaN(scNum)) { filteredProducts = filteredProducts.filter(product => product.brand_id === scNum || product.categories.some(cat => cat.id === scNum) ); } // Apply Brand Filter (b) if (bNum !== undefined && !isNaN(bNum)) { console.log(`Applying Brand Filter: brand_id === ${bNum}`); const preFilterCount = filteredProducts.length; allBrands = allBrands.filter(brand => brand.category_id === bNum) filteredProducts = filteredProducts.filter(product => allBrands.some(brand => brand.id === product.brand_id) ); console.log(`Filtered from ${preFilterCount} to ${filteredProducts.length} products after applying brand filter.`); } // Apply Price Range Filter if (minPriceANum > 0 || maxPriceANum < Infinity) { filteredProducts = filteredProducts.filter(product => { const productPrices = product.sizes.map(size => size.price); if (productPrices.length === 0) return false; const productMinPrice = Math.min(...productPrices); const productMaxPrice = Math.max(...productPrices); return productMinPrice >= minPriceANum && productMaxPrice <= maxPriceANum; }); } // Apply Availability Filter if (availability === 'in_stock') { filteredProducts = filteredProducts.filter(product => product.sizes.some(size => size.qty > 0) ); } else if (availability === 'out_of_stock') { filteredProducts = filteredProducts.filter(product => product.sizes.every(size => size.qty === 0) ); } // Log total after filtering console.log(`Total products after filtering: ${filteredProducts.length}`); // Apply Sorting if (sortBy && typeof sortBy === 'string') { switch (sortBy) { case 'Price: Low to High': filteredProducts.sort((a, b) => { const aMin = Math.min(...a.sizes.map(s => s.price)); const bMin = Math.min(...b.sizes.map(s => s.price)); return aMin - bMin; }); break; case 'Price: High to Low': filteredProducts.sort((a, b) => { const aMax = Math.max(...a.sizes.map(s => s.price)); const bMax = Math.max(...b.sizes.map(s => s.price)); return bMax - aMax; }); break; case 'Name: A to Z': filteredProducts.sort((a, b) => a.name.localeCompare(b.name)); break; case 'Name: Z to A': filteredProducts.sort((a, b) => b.name.localeCompare(a.name)); break; case 'Best Selling': filteredProducts.sort((a, b) => b.sold - a.sold); break; case 'Newest Arrivals': filteredProducts.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); break; default: // If sortBy parameter doesn't match any case, do not sort break; } } // Apply Pagination const totalFilteredProducts = filteredProducts.length; const totalPages = Math.ceil(totalFilteredProducts / itemsPerPageNum); const startIdx = (pageNum - 1) * itemsPerPageNum; const endIdx = startIdx + itemsPerPageNum; const paginatedProducts = filteredProducts.slice(startIdx, endIdx); // Construct Response const response = { products: paginatedProducts, total: totalFilteredProducts, page: pageNum, itemsPerPage: itemsPerPageNum, totalPages, hasMore: endIdx < totalFilteredProducts, minPrice, // Global minPrice across all products maxPrice, // Global maxPrice across all products }; // Send Response res.json(response);} catch (error) { console.error('Error fetching products:', error); res.status(500).json({ error: 'Internal server error' }); } finally { // Optional: Any cleanup operations can be performed here // For example, monitoring memory usage or logging request completion } }; cacheHelper.ts: // src/cacheHelpers.ts
import { productCache } from "./cache"; import { Product, CacheData, Color, Size } from "./types/product"; // Adjust the import path as necessary
/**
-
Adds a new product to the cache. */ export const addProductToCache = (newProduct: Product) => { try { const cacheData: CacheData | undefined = productCache.get('allProducts'); if (cacheData) { cacheData.allProducts.push(newProduct);
// Extract all prices const productPrices: number[] = []; productPrices.push(newProduct.price); newProduct.colors.forEach((color: Color) => { productPrices.push(color.price); color.sizes.forEach((size: Size) => productPrices.push(size.price)); }); newProduct.sizes.forEach((size: Size) => productPrices.push(size.price)); // Update minPrice and maxPrice if (productPrices.length > 0) { const productMinPrice = Math.min(...productPrices, cacheData.minPrice); const productMaxPrice = Math.max(...productPrices, cacheData.maxPrice); cacheData.minPrice = Math.min(cacheData.minPrice, productMinPrice); cacheData.maxPrice = Math.max(cacheData.maxPrice, productMaxPrice); } productCache.set('allProducts', cacheData); console.log('Added new product to cache.'); } else { // Flush cache to force reload on next access productCache.del('allProducts'); console.log('Cache unavailable. Flushed product cache.'); }} catch (error) { console.error('Error adding product to cache:', error); productCache.del('allProducts'); } };
/**
-
Updates an existing product in the cache. */ export const updateProductInCache = (updatedProduct: Product) => { try { const cacheData: CacheData | undefined = productCache.get('allProducts'); if (cacheData) { const index = cacheData.allProducts.findIndex(p => p.id === updatedProduct.id); if (index !== -1) { cacheData.allProducts[index] = updatedProduct;
// Recalculate minPrice and maxPrice const allPrices: number[] = []; cacheData.allProducts.forEach(product => { allPrices.push(product.price); product.colors.forEach(color => { allPrices.push(color.price); color.sizes.forEach(size => allPrices.push(size.price)); }); product.sizes.forEach(size => allPrices.push(size.price)); }); if (allPrices.length > 0) { cacheData.minPrice = Math.min(...allPrices); cacheData.maxPrice = Math.max(...allPrices); } else { cacheData.minPrice = 0; cacheData.maxPrice = 0; } productCache.set('allProducts', cacheData); console.log('Updated product in cache.'); } else { // Product not found; flush cache productCache.del('allProducts'); console.log('Product not found in cache. Flushed product cache.'); } } else { // Flush cache to force reload productCache.del('allProducts'); console.log('Cache unavailable. Flushed product cache.'); }} catch (error) { console.error('Error updating product in cache:', error); productCache.del('allProducts'); } };
/**
-
Deletes a product from the cache. */ export const deleteProductFromCache = (productId: number) => { try { const cacheData: CacheData | undefined = productCache.get('allProducts'); if (cacheData) { const initialLength = cacheData.allProducts.length; cacheData.allProducts = cacheData.allProducts.filter(p => p.id !== productId); if (cacheData.allProducts.length < initialLength) { // Recalculate minPrice and maxPrice const allPrices: number[] = []; cacheData.allProducts.forEach(product => { allPrices.push(product.price); product.colors.forEach(color => { allPrices.push(color.price); color.sizes.forEach(size => allPrices.push(size.price)); }); product.sizes.forEach(size => allPrices.push(size.price)); });
if (allPrices.length > 0) { cacheData.minPrice = Math.min(...allPrices); cacheData.maxPrice = Math.max(...allPrices); } else { cacheData.minPrice = 0; cacheData.maxPrice = 0; } productCache.set('allProducts', cacheData); console.log('Deleted product from cache.'); } else { // Product not found; flush cache productCache.del('allProducts'); console.log('Product not found in cache. Flushed product cache.'); } } else { // Flush cache to force reload productCache.del('allProducts'); console.log('Cache unavailable. Flushed product cache.'); }} catch (error) { console.error('Error deleting product from cache:', error); productCache.del('allProducts'); } };
/**
-
Decreases product, size, or color quantity in the cache based on identifiers.
-
@param productId - ID of the product
-
@param sizeId - ID of the size (optional)
-
@param colorId - ID of the color (optional)
-
@param quantity - Quantity to decrease */ export const decreaseProductQtyInCache = (productId: number, sizeId: number | null, colorId: number | null, quantity: number) => { try { const cacheData: CacheData | undefined = productCache.get('allProducts'); if (!cacheData) { // Cache is unavailable; nothing to update return; }
const product = cacheData.allProducts.find(p => p.id === productId); if (!product) { // Product not found in cache; nothing to update return; } if (sizeId) { // Decrease quantity in the specific size const size = product.sizes.find(s => s.id === sizeId); if (size) { size.qty = size.qty - quantity; if (size.qty < 0) size.qty = 0; // Prevent negative stock } // Also, if color_id is present, update the associated color's size if (colorId) { const color = product.colors.find(c => c.id === colorId); if (color) { const colorSize = color.sizes.find(s => s.id === sizeId); if (colorSize) { colorSize.qty = colorSize.qty - quantity; if (colorSize.qty < 0) colorSize.qty = 0; } } } } else if (colorId) { // Decrease quantity in the specific color const color = product.colors.find(c => c.id === colorId); if (color) { color.qty = color.qty - quantity; if (color.qty < 0) color.qty = 0; // Prevent negative stock } } else { // Decrease quantity in the product itself product.qty = product.qty - quantity; if (product.qty < 0) product.qty = 0; // Prevent negative stock } // Update the cache productCache.set('allProducts', cacheData); console.log('Decreased product quantity in cache.');} catch (error) { console.error('Error decreasing product quantity in cache:', error); // Decide whether to flush the cache or handle the error differently productCache.del('allProducts'); } };
/**
-
Increases the 'sold' count for a size in the cache. */ export const increaseSizeSoldInCache = (productId: number, sizeId: number, quantity: number): void => { try { const cacheData: CacheData | undefined = productCache.get('allProducts'); if (!cacheData) { // Cache is unavailable; nothing to update return; }
const product = cacheData.allProducts.find(p => p.id === productId); if (!product) { // Product not found in cache; nothing to update return; } const size = product.sizes.find(s => s.id === sizeId); if (size) { size.sold += quantity; // Ensure cache is updated productCache.set('allProducts', cacheData); console.log(`Increased sold count for size ID ${sizeId} in cache.`); }} catch (error) { console.error('Error increasing size sold in cache:', error); // Optionally, decide to flush the cache or handle the error differently productCache.del('allProducts'); } };
/**
-
Increases the 'sold' count for a color in the cache. */ export const increaseColorSoldInCache = (productId: number, colorId: number, quantity: number): void => { try { const cacheData: CacheData | undefined = productCache.get('allProducts'); if (!cacheData) { // Cache is unavailable; nothing to update return; }
const product = cacheData.allProducts.find(p => p.id === productId); if (!product) { // Product not found in cache; nothing to update return; } const color = product.colors.find(c => c.id === colorId); if (color) { color.sold += quantity; // Ensure cache is updated productCache.set('allProducts', cacheData); console.log(`Increased sold count for color ID ${colorId} in cache.`); }} catch (error) { console.error('Error increasing color sold in cache:', error); // Optionally, decide to flush the cache or handle the error differently productCache.del('allProducts'); } };
/**
-
Increases the 'sold' count for a product in the cache. */ export const increaseProductSoldInCache = (productId: number, quantity: number): void => { try { const cacheData: CacheData | undefined = productCache.get('allProducts'); if (!cacheData) { // Cache is unavailable; nothing to update return; }
const product = cacheData.allProducts.find(p => p.id === productId); if (product) { product.sold += quantity; // Ensure cache is updated productCache.set('allProducts', cacheData); console.log(`Increased sold count for product ID ${productId} in cache.`); }} catch (error) { console.error('Error increasing product sold in cache:', error); // Optionally, decide to flush the cache or handle the error differently productCache.del('allProducts'); } }; types/product.ts: // src/types/productInput.ts
export interface ImageInput { url: string; color_id?: number | null; size_id?: number | null; }
export interface SizeInput { id?: number; // Optional for creation name: string; price: number; // Changed from string to number qty: number; sold?: number; color_id?: number | null; images: ImageInput[]; }
export interface ColorInput { id?: number; // Optional for creation name: string; price: number; // Changed from string to number qty: number; sold?: number; has_size: boolean; sizes: SizeInput[]; images: ImageInput[]; }
export interface CreateProductInput { name: string; description: string; brand_id: number; has_color: boolean; has_size: boolean; product_type: string; price: number; // Changed from string to number qty: number; category_ids: number[]; images: ImageInput[]; colors?: ColorInput[]; // Optional based on has_color sizes?: SizeInput[]; // Optional based on has_size and has_color }
export interface UpdateProductInput extends CreateProductInput {}
export interface Category { id: number; name: string; main_category_id: number; main_category_name: string; }
export interface Image { id: number; url: string; color_id: number | null; size_id: number | null; }
export interface Size { id: number; name: string; price: number; // Changed from string to number for arithmetic operations qty: number; sold: number; color_id: number | null; images: string[]; }
export interface Color { id: number; name: string; price: number; // Changed from string to number qty: number; sold: number; has_size: boolean; // Mandatory property sizes: Size[]; images: string[]; }
export interface Product { id: number; name: string; description: string; brand_id: number; // Also a subcategory_id for brands has_color: boolean; has_size: boolean; product_type: string; price: number; // Changed from string to number qty: number; sold: number; created_at: string; updated_at: string; categories: Category[]; colors: Color[]; sizes: Size[]; images: Image[]; }
export interface Brand { id: number; name: string; category_id: number; category_name: string; }
export interface CacheData { allProducts: Product[]; minPrice: number; maxPrice: number; }
export interface BrandsCacheData { allBrands: Brand[]; } server.ts: import express from 'express'; import morgan from 'morgan'; import dotenv from 'dotenv'; import cors from 'cors'; import compression from 'compression'; import { errorHandler } from './middleware/errorHandler'; import categoryRoutes from './routes/categoryRoutes'; import subcategoryRoutes from './routes/subcategoryRoutes'; import productRoutes from "./routes/productRoutes"; import userRoutes from "./routes/userRoutes"; import addressRoutes from "./routes/addressRoutes"; import couponRoutes from "./routes/couponRoutes"; import cartRoutes from "./routes/cartRoutes"; import wishlistRoutes from "./routes/wishlistRoutes"; import nocache from 'nocache'; import counterRoutes from "./routes/counterRoutes"; import imageOptimizationRoutes from "./routes/imageOptimizationRoutes";
dotenv.config();
const app = express();
app.use(cors()); app.use(compression()); app.use(nocache()); app.use(express.json({ limit: '500kb' })); app.use(morgan('dev'));
app.set('trust proxy', 1); app.use('/user', userRoutes); app.use('/categories', categoryRoutes); app.use('/subcategories', subcategoryRoutes); app.use('/products', productRoutes); app.use('/addresses', addressRoutes); app.use('/coupons', couponRoutes); app.use('/counter', counterRoutes); app.use('/cart', cartRoutes); app.use('/wishlist', wishlistRoutes); app.use('/images', imageOptimizationRoutes); app.use(errorHandler);
const PORT = 4000; app.listen(PORT, () => { console.log(
Server running on port ${PORT}); });export default app;
-
- Assistant
To optimize the
getProductsperformance, enhance its stability, improve filtering and searching, and removeconsole.logstatements while ensuring it operates efficiently within a 1GB RAM VPS environment, we can undertake several key improvements:- Cache Key Consistency: Ensure consistent cache keys to prevent cache misses.
- Efficient Data Structures: Utilize
Mapfor faster lookups where applicable. - Optimized Filtering: Combine multiple filtering steps to reduce the number of iterations over the data.
- Lazy Loading: Load brands only when necessary to save memory.
- Remove
console.log: Eliminateconsole.logstatements to reduce unnecessary I/O operations. - Error Handling: Enhance error handling to maintain stability.
- Pagination Early Exit: Apply pagination as early as possible to limit the amount of data processed.
Below is the revised and optimized version of your
productController.ts, along with necessary adjustments tocache.tsand other relevant files.
cache.tsEnsure that your cache module uses consistent keys and leverages efficient caching mechanisms. Here’s an optimized version using the
node-cachelibrary:// src/cache.ts import NodeCache from 'node-cache'; import { CacheData, BrandsCacheData } from './types/product'; // Initialize caches with appropriate TTL (Time-To-Live) // Adjust TTL as per your application's requirements export const productCache = new NodeCache<{ allProducts: CacheData }>({ stdTTL: 3600, // 1 hour checkperiod: 120, }); export const brandsCache = new NodeCache<{ allBrands: BrandsCacheData }>({ stdTTL: 3600, // 1 hour checkperiod: 120, });Installation:
Make sure to install
node-cacheif you haven't already:npm install node-cache
productController.tsHere's the optimized
getProductscontroller with all the required improvements:// src/controllers/productController.ts import { Request, Response } from 'express'; import { query } from '../config/database'; import { brandsCache, productCache } from '../cache'; import { Brand, BrandsCacheData, CacheData, Product, } from '../types/product'; import { loadAndCacheAllProducts, loadAndCacheAllBrands, } from '../services/cacheService'; // Move cache loading functions to services /** * Controller to handle fetching and responding with filtered products. * Caches all products and brands on first request and performs in-memory filtering thereafter. * @param {Request} req - Express request object containing query parameters. * @param {Response} res - Express response object to send JSON data. */ export const getProducts = async (req: Request, res: Response): Promise<void> => { try { // Retrieve cached products let cacheData: CacheData | undefined = productCache.get('allProducts')?.allProducts; if (!cacheData) { // Cache miss: Load from DB and cache cacheData = await loadAndCacheAllProducts(); } const { allProducts, minPrice, maxPrice } = cacheData; // Extract and sanitize query parameters const { search = '', c, sc, b, minPriceA = '0', maxPriceA = '', availability = 'all', sortBy = '', page = '1', itemsPerPage = '15', } = req.query; // Parse numerical values with fallback const pageNum = Math.max(parseInt(page as string, 10) || 1, 1); const itemsPerPageNum = Math.max(parseInt(itemsPerPage as string, 10) || 15, 1); const minPriceANum = parseFloat(minPriceA as string) || 0; let maxPriceANum = maxPriceA ? parseFloat(maxPriceA as string) : Infinity; // Sanitize maxPriceA if (!isNaN(maxPriceANum) && maxPriceANum < minPriceANum) { maxPriceANum = Infinity; } // Parse c, sc, b as numbers if present const cNum = c ? parseInt(c as string, 10) : undefined; const scNum = sc ? parseInt(sc as string, 10) : undefined; const bNum = b ? parseInt(b as string, 10) : undefined; let allBrands: Brand[] = []; if (bNum !== undefined && !isNaN(bNum)) { // Retrieve cached brands const brandsCacheData = brandsCache.get('allBrands')?.allBrands; if (!brandsCacheData) { // Cache miss: Load from DB and cache allBrands = await loadAndCacheAllBrands(); } else { allBrands = brandsCacheData; } // Validate brand const isValidBrand = allBrands.some(brand => brand.category_id === bNum); if (!isValidBrand) { res.json({ products: [], total: 0, page: pageNum, itemsPerPage: itemsPerPageNum, totalPages: 0, hasMore: false, minPrice, maxPrice, }); return; } } // Start with all products let filteredProducts = allProducts; // Apply Search, Category, Subcategory, and Brand Filters in a single pass if (search || cNum !== undefined || scNum !== undefined || bNum !== undefined) { const searchLower = typeof search === 'string' ? search.toLowerCase() : ''; const searchWords = searchLower.split(/\s+/).filter(Boolean); filteredProducts = allProducts.filter(product => { // Search Filter if (searchWords.length > 0) { const name = product.name.toLowerCase(); if (!searchWords.every(word => name.includes(word))) { return false; } } // Category Filter if (cNum !== undefined) { const hasCategory = product.categories.some(cat => cat.main_category_id === cNum); if (!hasCategory) return false; } // Subcategory Filter if (scNum !== undefined) { const hasSubcategory = product.brand_id === scNum || product.categories.some(cat => cat.id === scNum); if (!hasSubcategory) return false; } // Brand Filter if (bNum !== undefined) { const belongsToBrand = allBrands.some(brand => brand.id === product.brand_id); if (!belongsToBrand) return false; } return true; }); } // Apply Price Range Filter if (minPriceANum > 0 || maxPriceANum < Infinity) { filteredProducts = filteredProducts.filter(product => { let productPriceMin = product.price; let productPriceMax = product.price; if (product.has_color) { product.colors.forEach(color => { productPriceMin = Math.min(productPriceMin, color.price); productPriceMax = Math.max(productPriceMax, color.price); color.sizes.forEach(size => { productPriceMin = Math.min(productPriceMin, size.price); productPriceMax = Math.max(productPriceMax, size.price); }); }); } if (product.has_size && !product.has_color) { product.sizes.forEach(size => { productPriceMin = Math.min(productPriceMin, size.price); productPriceMax = Math.max(productPriceMax, size.price); }); } return productPriceMin >= minPriceANum && productPriceMax <= maxPriceANum; }); } // Apply Availability Filter if (availability === 'in_stock') { filteredProducts = filteredProducts.filter(product => { if (product.has_color) { return product.colors.some(color => color.sizes.some(size => size.qty > 0) ); } else if (product.has_size) { return product.sizes.some(size => size.qty > 0); } else { return product.qty > 0; } }); } else if (availability === 'out_of_stock') { filteredProducts = filteredProducts.filter(product => { if (product.has_color) { return product.colors.every(color => color.sizes.every(size => size.qty === 0) ); } else if (product.has_size) { return product.sizes.every(size => size.qty === 0); } else { return product.qty === 0; } }); } // Apply Sorting if (typeof sortBy === 'string' && sortBy.trim() !== '') { switch (sortBy) { case 'Price: Low to High': filteredProducts.sort((a, b) => { const aMin = a.has_color ? Math.min(...a.colors.flatMap(color => [color.price, ...color.sizes.map(s => s.price)])) : a.has_size ? Math.min(...a.sizes.map(s => s.price)) : a.price; const bMin = b.has_color ? Math.min(...b.colors.flatMap(color => [color.price, ...color.sizes.map(s => s.price)])) : b.has_size ? Math.min(...b.sizes.map(s => s.price)) : b.price; return aMin - bMin; }); break; case 'Price: High to Low': filteredProducts.sort((a, b) => { const aMax = a.has_color ? Math.max(...a.colors.flatMap(color => [color.price, ...color.sizes.map(s => s.price)])) : a.has_size ? Math.max(...a.sizes.map(s => s.price)) : a.price; const bMax = b.has_color ? Math.max(...b.colors.flatMap(color => [color.price, ...color.sizes.map(s => s.price)])) : b.has_size ? Math.max(...b.sizes.map(s => s.price)) : b.price; return bMax - aMax; }); break; case 'Name: A to Z': filteredProducts.sort((a, b) => a.name.localeCompare(b.name)); break; case 'Name: Z to A': filteredProducts.sort((a, b) => b.name.localeCompare(a.name)); break; case 'Best Selling': filteredProducts.sort((a, b) => b.sold - a.sold); break; case 'Newest Arrivals': filteredProducts.sort( (a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime() ); break; // Add more sort options as needed default: break; } } // Calculate Pagination const totalFilteredProducts = filteredProducts.length; const totalPages = Math.ceil(totalFilteredProducts / itemsPerPageNum); const startIdx = (pageNum - 1) * itemsPerPageNum; const endIdx = startIdx + itemsPerPageNum; const paginatedProducts = filteredProducts.slice(startIdx, endIdx); // Construct Response const response = { products: paginatedProducts, total: totalFilteredProducts, page: pageNum, itemsPerPage: itemsPerPageNum, totalPages, hasMore: endIdx < totalFilteredProducts, minPrice, maxPrice, }; // Send Response res.json(response); } catch (error) { // Handle errors appropriately res.status(500).json({ error: 'Internal server error' }); } };Key Improvements:
-
Cache Key Consistency:
- Before: The cache was being accessed with different keys (
'cacheData'vs'allProducts'). - After: Consistently using
'allProducts'as the cache key.
- Before: The cache was being accessed with different keys (
-
Efficient Filtering:
- Combined multiple filters into a single
.filterpass to reduce the number of iterations over the products array. - Utilized efficient array methods like
flatMapto flatten nested arrays when calculating min and max prices.
- Combined multiple filters into a single
-
Removed
console.logStatements:- All
console.logandconsole.warnstatements have been removed to reduce I/O overhead.
- All
-
Optimized Sorting:
- Used
flatMapto flatten nested arrays when calculating prices for sorting, which is more efficient. - Ensured sorting operations are only performed when necessary.
- Used
-
Enhanced Error Handling:
- Simplified error handling by catching errors and responding with a 500 status code without exposing sensitive information.
-
Pagination Early Exit:
- Applied pagination after all filtering and sorting to limit the amount of data sent in the response.
cacheService.tsTo further modularize the code, move cache loading functions to a separate service:
// src/services/cacheService.ts import { query } from '../config/database'; import { brandsCache, productCache } from '../cache'; import { Brand, BrandsCacheData, CacheData, Product, } from '../types/product'; /** * Loads all products from the database, processes them, and caches the result. * @returns {Promise<CacheData>} The cached product data. */ export const loadAndCacheAllProducts = async (): Promise<CacheData> => { const q = ` SELECT p.id, p.name, p.description, p.brand_id, p.has_color, p.has_size, p.product_type, p.price, p.qty, p.sold, p.created_at, p.updated_at, sc.id AS subcategory_id, sc.name AS subcategory_name, c.id AS main_category_id, c.name AS main_category_name, s.id AS size_id, s.name AS size_name, s.price AS size_price, s.qty AS size_qty, s.sold AS size_sold, cl.id AS color_id, cl.name AS color_name, cl.price AS color_price, cl.qty AS color_qty, cl.sold AS color_sold, i.id AS image_id, i.url AS image_url, i.color_id AS image_color_id, i.size_id AS image_size_id FROM products p LEFT JOIN product_categories pc ON p.id = pc.product_id LEFT JOIN subcategories sc ON pc.category_id = sc.id LEFT JOIN categories c ON sc.category_id = c.id LEFT JOIN colors cl ON p.id = cl.product_id LEFT JOIN sizes s ON p.id = s.product_id LEFT JOIN images i ON (p.id = i.product_id AND (cl.id = i.color_id OR i.color_id IS NULL) AND (s.id = i.size_id OR i.size_id IS NULL)) `; try { const result = await query(q, []); const productsMap: Map<number, Product> = new Map(); result.rows.forEach((row: any) => { const productId = Number(row.id); let product = productsMap.get(productId); if (!product) { product = { id: productId, name: row.name, description: row.description, brand_id: Number(row.brand_id), has_color: Boolean(row.has_color), has_size: Boolean(row.has_size), product_type: row.product_type, price: Number(row.price), qty: Number(row.qty), sold: Number(row.sold), created_at: new Date(row.created_at).toISOString(), updated_at: new Date(row.updated_at).toISOString(), categories: [], colors: [], sizes: [], images: [], }; productsMap.set(productId, product); } // Handle Categories if (row.subcategory_id && row.main_category_id) { const categoryId = Number(row.subcategory_id); if (!product.categories.some(cat => cat.id === categoryId)) { product.categories.push({ id: categoryId, name: row.subcategory_name, main_category_id: Number(row.main_category_id), main_category_name: row.main_category_name, }); } } // Handle Colors if (row.color_id && row.color_name) { const colorId = Number(row.color_id); let color = product.colors.find(c => c.id === colorId); if (!color) { color = { id: colorId, name: row.color_name, price: Number(row.color_price), qty: Number(row.color_qty), sold: Number(row.color_sold), has_size: Boolean(row.has_size), sizes: [], images: [], }; product.colors.push(color); } // Handle Sizes within Colors if (row.size_id && row.size_name && product.has_color) { const sizeId = Number(row.size_id); if (!color.sizes.some(s => s.id === sizeId)) { color.sizes.push({ id: sizeId, name: row.size_name, price: Number(row.size_price), qty: Number(row.size_qty), sold: Number(row.size_sold), color_id: colorId, images: [], }); } } } // Handle Sizes without Colors if (row.size_id && row.size_name && !product.has_color && product.has_size) { const sizeId = Number(row.size_id); if (!product.sizes.some(s => s.id === sizeId)) { product.sizes.push({ id: sizeId, name: row.size_name, price: Number(row.size_price), qty: Number(row.size_qty), sold: Number(row.size_sold), color_id: null, images: [], }); } } // Handle Images if (row.image_id && row.image_url) { const imageId = Number(row.image_id); if (!product.images.some(img => img.id === imageId)) { product.images.push({ id: imageId, url: row.image_url, color_id: row.image_color_id ? Number(row.image_color_id) : null, size_id: row.image_size_id ? Number(row.image_size_id) : null, }); } // Associate Images with Colors or Sizes if (row.image_color_id) { const color = product.colors.find(c => c.id === Number(row.image_color_id)); if (color && !color.images.includes(row.image_url)) { color.images.push(row.image_url); } } if (row.image_size_id) { const sizeId = Number(row.image_size_id); if (product.has_color) { product.colors.forEach(color => { const size = color.sizes.find(s => s.id === sizeId); if (size && !size.images.includes(row.image_url)) { size.images.push(row.image_url); } }); } else if (product.has_size) { const size = product.sizes.find(s => s.id === sizeId); if (size && !size.images.includes(row.image_url)) { size.images.push(row.image_url); } } } } }); const allProducts = Array.from(productsMap.values()); // Calculate min and max prices let minPrice = Infinity; let maxPrice = -Infinity; allProducts.forEach(product => { const productPrices = [product.price]; if (product.has_color) { product.colors.forEach(color => { productPrices.push(color.price); color.sizes.forEach(size => { productPrices.push(size.price); }); }); } if (product.has_size && !product.has_color) { product.sizes.forEach(size => { productPrices.push(size.price); }); } const productMin = Math.min(...productPrices); const productMax = Math.max(...productPrices); if (productMin < minPrice) minPrice = productMin; if (productMax > maxPrice) maxPrice = productMax; }); // Fallback values if (minPrice === Infinity) minPrice = 0; if (maxPrice === -Infinity) maxPrice = 0; const cacheData: CacheData = { allProducts, minPrice, maxPrice }; productCache.set('allProducts', { allProducts, minPrice, maxPrice }); return cacheData; } catch (error) { throw new Error('Error loading products from DB'); } }; /** * Fetches all brands from the database and caches them. * @returns {Promise<Brand[]>} A promise that resolves to the cached brand data. */ export const loadAndCacheAllBrands = async (): Promise<Brand[]> => { const q = ` SELECT s.*, c.name as category_name FROM subcategories s JOIN categories c ON s.category_id = c.id AND c.is_brand ORDER BY c.name, s.name `; try { const result = await query(q, []); const allBrands: Brand[] = result.rows.map((row: any) => ({ id: Number(row.id), name: row.name, category_id: Number(row.category_id), category_name: row.category_name, })); // Cache the brands brandsCache.set('allBrands', { allBrands }); return allBrands; } catch (error) { throw new Error('Error loading brands from DB'); } };Key Improvements:
-
Modularization: Moved cache loading functions (
loadAndCacheAllProductsandloadAndCacheAllBrands) to a separatecacheService.tswithin theservicesdirectory for better organization and separation of concerns. -
Use of Maps: Utilized
Mapto store products temporarily during processing for faster access and to prevent duplicates. -
Optimized Min/Max Price Calculation: Streamlined the calculation of
minPriceandmaxPriceby aggregating all relevant prices in a single pass. -
Error Handling: Simplified error messages to avoid exposing sensitive information.
types/product.tsEnsure that your TypeScript types are correctly defined. Here's a slightly refined version:
// src/types/product.ts export interface ImageInput { url: string; color_id?: number | null; size_id?: number | null; } export interface SizeInput { id?: number; // Optional for creation name: string; price: number; qty: number; sold?: number; color_id?: number | null; images: ImageInput[]; } export interface ColorInput { id?: number; // Optional for creation name: string; price: number; qty: number; sold?: number; has_size: boolean; sizes: SizeInput[]; images: ImageInput[]; } export interface CreateProductInput { name: string; description: string; brand_id: number; has_color: boolean; has_size: boolean; product_type: string; price: number; qty: number; category_ids: number[]; images: ImageInput[]; colors?: ColorInput[]; sizes?: SizeInput[]; } export interface UpdateProductInput extends CreateProductInput {} export interface Category { id: number; name: string; main_category_id: number; main_category_name: string; } export interface Image { id: number; url: string; color_id: number | null; size_id: number | null; } export interface Size { id: number; name: string; price: number; qty: number; sold: number; color_id: number | null; images: string[]; } export interface Color { id: number; name: string; price: number; qty: number; sold: number; has_size: boolean; sizes: Size[]; images: string[]; } export interface Product { id: number; name: string; description: string; brand_id: number; has_color: boolean; has_size: boolean; product_type: string; price: number; qty: number; sold: number; created_at: string; updated_at: string; categories: Category[]; colors: Color[]; sizes: Size[]; images: Image[]; } export interface Brand { id: number; name: string; category_id: number; category_name: string; } export interface CacheData { allProducts: Product[]; minPrice: number; maxPrice: number; } export interface BrandsCacheData { allBrands: Brand[]; }
cacheHelper.tsSince you've moved cache-related operations to
cacheService.ts, you can remove redundant logic fromcacheHelper.ts. IfcacheHelper.tscontains essential functions for adding, updating, or deleting products from the cache, consider optimizing them similarly. Here's an example of optimizing the cache helper functions:// src/helpers/cacheHelper.ts import { productCache, brandsCache } from '../cache'; import { Product, Color, Size, CacheData } from '../types/product'; /** * Adds a new product to the cache. */ export const addProductToCache = (newProduct: Product) => { try { const cacheData = productCache.get('allProducts'); if (cacheData) { cacheData.allProducts.push(newProduct); // Update minPrice and maxPrice const productPrices = [newProduct.price]; if (newProduct.has_color) { newProduct.colors.forEach(color => { productPrices.push(color.price); color.sizes.forEach(size => productPrices.push(size.price)); }); } if (newProduct.has_size && !newProduct.has_color) { newProduct.sizes.forEach(size => productPrices.push(size.price)); } cacheData.minPrice = Math.min(cacheData.minPrice, ...productPrices); cacheData.maxPrice = Math.max(cacheData.maxPrice, ...productPrices); productCache.set('allProducts', cacheData); } else { // If cache is unavailable, trigger a reload productCache.del('allProducts'); } } catch { // On error, invalidate the cache to force a reload productCache.del('allProducts'); } }; /** * Updates an existing product in the cache. */ export const updateProductInCache = (updatedProduct: Product) => { try { const cacheData = productCache.get('allProducts'); if (cacheData) { const index = cacheData.allProducts.findIndex(p => p.id === updatedProduct.id); if (index !== -1) { cacheData.allProducts[index] = updatedProduct; // Recalculate minPrice and maxPrice const allPrices = cacheData.allProducts.flatMap(product => { let prices = [product.price]; if (product.has_color) { product.colors.forEach(color => { prices.push(color.price); prices = prices.concat(color.sizes.map(s => s.price)); }); } if (product.has_size && !product.has_color) { prices = prices.concat(product.sizes.map(s => s.price)); } return prices; }); cacheData.minPrice = Math.min(...allPrices, 0); cacheData.maxPrice = Math.max(...allPrices, 0); productCache.set('allProducts', cacheData); } else { // If product not found, invalidate cache productCache.del('allProducts'); } } } catch { productCache.del('allProducts'); } }; /** * Deletes a product from the cache. */ export const deleteProductFromCache = (productId: number) => { try { const cacheData = productCache.get('allProducts'); if (cacheData) { const initialLength = cacheData.allProducts.length; cacheData.allProducts = cacheData.allProducts.filter(p => p.id !== productId); if (cacheData.allProducts.length < initialLength) { // Recalculate minPrice and maxPrice const allPrices = cacheData.allProducts.flatMap(product => { let prices = [product.price]; if (product.has_color) { product.colors.forEach(color => { prices.push(color.price); prices = prices.concat(color.sizes.map(s => s.price)); }); } if (product.has_size && !product.has_color) { prices = prices.concat(product.sizes.map(s => s.price)); } return prices; }); cacheData.minPrice = Math.min(...allPrices, 0); cacheData.maxPrice = Math.max(...allPrices, 0); productCache.set('allProducts', cacheData); } else { // If product not found, invalidate cache productCache.del('allProducts'); } } } catch { productCache.del('allProducts'); } };Key Improvements:
- Simplified Price Updates: Utilized
flatMapto flatten arrays while calculating prices forminPriceandmaxPrice. - Error Handling: Any error during cache manipulation results in invalidating the cache to ensure consistency.
- Avoid Redundant Operations: Removed unnecessary checks and streamlined the logic.
server.tsNo changes are required here regarding the optimizations. However, ensure that the server uses the updated
getProductscontroller.// src/server.ts import express from 'express'; import morgan from 'morgan'; import dotenv from 'dotenv'; import cors from 'cors'; import compression from 'compression'; import nocache from 'nocache'; import { errorHandler } from './middleware/errorHandler'; import categoryRoutes from './routes/categoryRoutes'; import subcategoryRoutes from './routes/subcategoryRoutes'; import productRoutes from './routes/productRoutes'; import userRoutes from './routes/userRoutes'; import addressRoutes from './routes/addressRoutes'; import couponRoutes from './routes/couponRoutes'; import cartRoutes from './routes/cartRoutes'; import wishlistRoutes from './routes/wishlistRoutes'; import counterRoutes from './routes/counterRoutes'; import imageOptimizationRoutes from './routes/imageOptimizationRoutes'; dotenv.config(); const app = express(); app.use(cors()); app.use(compression()); app.use(nocache()); app.use(express.json({ limit: '500kb' })); app.use(morgan('tiny')); // Changed to 'tiny' to reduce logging overhead app.set('trust proxy', 1); app.use('/user', userRoutes); app.use('/categories', categoryRoutes); app.use('/subcategories', subcategoryRoutes); app.use('/products', productRoutes); app.use('/addresses', addressRoutes); app.use('/coupons', couponRoutes); app.use('/counter', counterRoutes); app.use('/cart', cartRoutes); app.use('/wishlist', wishlistRoutes); app.use('/images', imageOptimizationRoutes); app.use(errorHandler); const PORT = process.env.PORT || 4000; app.listen(PORT, () => { // Optionally, use a proper logging library here // Removed console.log as per requirements }); export default app;Key Changes:
-
Morgan Logging: Changed the Morgan logging format to
'tiny'to reduce the amount of logging and conserve I/O resources. -
Removed
console.log: The server's startup log (console.log) has been removed to adhere to your requirement to eliminateconsole.logstatements. If logging is necessary, consider integrating a logging library likewinstonwith appropriate log levels. -
Environment Variables: Updated the
PORTto be configurable via environment variables for better flexibility.
Additional Recommendations
-
Use a Proper Logging Library: Instead of using
console.log, consider using a logging library likewinstonorpino. These libraries offer better performance and more control over log levels and outputs. -
Monitor Memory Usage: Given the 1GB RAM constraint, implement monitoring to track memory usage and cache sizes. Tools like
node-memwatchcan be integrated for this purpose. -
Implement Cache Eviction Policies: Depending on the size of your product dataset, consider implementing cache eviction policies to prevent excessive memory usage. Libraries like
node-cachesupport TTL and other eviction strategies. -
Optimize Database Indexes: Ensure that your database has appropriate indexes on fields used in filtering (e.g.,
brand_id,category_id,price, etc.) to speed up initial data loading. -
Lazy Load Brands: If brands are rarely filtered upon, consider loading and caching brands only when the
bquery parameter is present. This reduces memory usage when brand filtering isn't required. -
API Rate Limiting: Implement rate limiting to prevent abuse and ensure the server remains responsive under high load. Libraries like
express-rate-limitcan be used. -
Pagination Limit: Enforce a maximum limit on
itemsPerPageto prevent excessively large responses that can consume memory and processing power. -
Asynchronous Operations: Ensure all asynchronous operations are handled correctly to prevent memory leaks or unhandled promise rejections.
-
Use TypeScript Strict Mode: Enable strict mode in TypeScript to catch potential bugs and ensure type safety across your application.
-
Code Profiling: Use profiling tools to identify and optimize any remaining bottlenecks in your application's performance.
By implementing these optimizations, your
getProductsendpoint should perform more efficiently, use memory effectively, maintain stability, and handle filtering and searching robustly within the constraints of a 1GB RAM VPS environment.