Tạp chí Thợ Code

npm create vite@latest

cd vite-project #your project name
npm install
npm run dev

npm install @nextui-org/react framer-motion
npm install -D tailwindcss postcss autoprefixer

node:internal/process/promises:394 triggerUncaughtException(err, true /* fromPromise */);
[Failed to load PostCSS config: Failed to load PostCSS config (searchPath: /home/checkvar/src/frontend): [ReferenceError] module is not defined in ES module scope
This file is being treated as an ES module because it has a '.js' file extension and '/home/checkvar/src/frontend/package.json' contains "type": "module". To treat it as a CommonJS script, rename it to use the '.cjs' file extension.
ReferenceError: module is not defined in ES module scope

VITE_APP_ENV=development
VITE_APP_TITLE="Font Bạt Detector v0.1 beta"
VITE_APP_API_BASE_URL=https://api.example.com

const { nextui } = require("@nextui-org/react");

/** @type {import('tailwindcss').Config} */
module.exports = {
  content: [
    "./index.html",
    "./*.{js,ts,jsx,tsx}",
    "./node_modules/@nextui-org/theme/dist/**/*.{js,ts,jsx,tsx}",
  ],
  theme: {
    extend: {},
  },
  darkMode: "class",
  plugins: [nextui()],
};

module.exports = {
  plugins: {
    tailwindcss: {},
    autoprefixer: {},
  },
}

@tailwind base;
@tailwind components;
@tailwind utilities;

import React, { useState, useEffect } from "react";
import {Table, TableHeader, TableColumn, TableBody, TableRow, TableCell, Pagination, Spinner, getKeyValue} from "@nextui-org/react";
import useSWR from "swr";
import {Input} from "@nextui-org/react";
import {Button, ButtonGroup} from "@nextui-org/button";

// const [searchTerm, setSearchTerm] = useState("");
const fetcher = (...args) => fetch(...args).then((res) => res.json());


export default function App() {
  const [page, setPage] = React.useState(1);
  const [searchTerm, setSearchTerm] = useState('');
  const [searchState, setSearchState] = useState('');
  const apiUrl = import.meta.env.VITE_APP_API_BASE_URL;
  // console.log('apiUrl', apiUrl);
  
  var dataUrl = `${apiUrl}?page=${page}`;
  // dataUrl = `https://swapi.py4e.com/api/people?page=${page}`;

  if (searchState == true && searchTerm != '') {
    dataUrl = `${apiUrl}?fontbat=${searchTerm}&page=${page}`;
    setSearchState(false);
  }

  var { data, isLoading } = useSWR(dataUrl, fetcher, {
    keepPreviousData: true,
  });

  const rowsPerPage = 30;

  var pages = React.useMemo(() => {
    return data?.count ? Math.ceil(data.count / rowsPerPage) : 0;
  }, [data?.count, rowsPerPage]);

  var loadingState = isLoading || data?.results.length === 0 ? "loading" : "idle";

  // console.log('dataresults',data.results);
  // console.log('url', window.location.href);
  var txtProcessBy = '';
    txtProcessBy = (<span style={{ fontSize: "small" }}>Xử lý bởi <a href="https://thocode.com" target="_blank">ThoCode.com</a> trên Dữ liệu của <a href="https://facebook.com/mttqvietnam" target="_blank" rel="nofollow noopener noreferrer">MTTQVN</a></span>)

  function test(){
    alert('Tính năng tìm kiếm hiện đang tắt');
  }

  const searchFunction = (event) => {
    console.log('e', event.target.value);
  }


  const handleSubmit = (event) => {
    event.preventDefault()
  }

  const handleKeyDown = (event) => {
    console.log('searchTerm', event.target.value);
    if(event.key === "Enter" && searchTerm != '')
    {
      setSearchState(true);
    }
  }

  useEffect(() => {
    document.title = import.meta.env.VITE_APP_TITLE;
  }, []); 

  return (

    <div style={{ width: "100%"}}>
      <h1>
        {import.meta.env.VITE_APP_TITLE}
        <span style={{ paddingLeft: 5, fontSize: "small", fontStyle: "italic"}}></span>
      </h1>
      {txtProcessBy}
      <div style={{ fontSize: "8pt"}}>
        <ul>
          <li><a href="/data/Thong tin ung ho qua TSK VCB 0011001932418 tu 01.09 den10.09.2024.zip#"></a>Thong tin ung ho qua TSK VCB 0011001932418 tu 01.09 den10.09.2024.pdf</li>
        </ul>
      </div>

      <div style={{ paddingBottom: 5 }} className="flex w-full flex-wrap md:flex-nowrap gap-4">
        <form action="/" onSubmit={handleSubmit}>
        <Input isClearable type="search" placeholder="Tìm kiếm"
          onClear={() => console.log("input cleared")}
          onChange={(e) => {
            // searchFunction(e);
            setSearchTerm(e.target.value);
            console.log("onchange")
          }}
          onKeyDown={handleKeyDown}
          // onValueChange={() => console.log("onvaluechange")}
          // onSubmit={() => console.log("onsubmit")}
        />

        </form>
      </div>


      <Table
        aria-label="Example table with client async pagination"
        bottomContent={
          pages > 0 ? (
            <div className="flex w-full justify-center">
              <Pagination
                isCompact
                showControls
                showShadow
                color="primary"
                page={page}
                total={pages}
                onChange={(page) => setPage(page)}
              />
            </div>
          ) : null
        }
      >
        <TableHeader>
          <TableColumn key="date">Ngày GD</TableColumn>
          <TableColumn key="code">Mã GD</TableColumn>
          <TableColumn key="amount">Số tiền</TableColumn>
          <TableColumn key="notes">Note</TableColumn>
        </TableHeader>
        <TableBody
          items={data?.results ?? []}
          loadingContent={<Spinner />}
          loadingState={loadingState}
        >
          {(item) => (
            <TableRow key={item?._id}>
              {(columnKey) => <TableCell>{getKeyValue(item, columnKey)}</TableCell>}
            </TableRow>
          )}
        </TableBody>
      </Table>

      
    </div>
  );
}

import React from "react";
import ReactDOM from "react-dom/client";
import { NextUIProvider } from "@nextui-org/react";
import App from "./App";
import "./styles.css";

ReactDOM.createRoot(document.getElementById("root")).render(
  <React.StrictMode>
    <NextUIProvider>
      <div style={{ padding: 10 }} className="w-screen  justify-center">
        <App />
      </div>  
    </NextUIProvider>
  </React.StrictMode>
);

npm run dev

npm run build

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "logs:PutLogEvents",
                "logs:CreateLogGroup",
                "logs:CreateLogStream"
            ],
            "Resource": "arn:aws:logs:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "s3:GetObject"
            ],
            "Resource": "arn:aws:s3:::*/*"
        }
    ]
}

# Phong Bat Detector 2

import boto3
import os
import sys
import uuid
from urllib.parse import unquote_plus
#from PIL import Image
#import PIL.Image

import os
import pdfplumber
import re
import PyPDF2
import json

from pymongo import MongoClient

print('Loading function')

s3_client = boto3.client('s3')

def split_pdf(pdf_path, output_dir, num_pages_per_file=100):
  """
  Hàm tách PDF thành nhiều file nhỏ hơn.

  Args:
      pdf_path: Đường dẫn đến tệp PDF.
      output_dir: Thư mục để lưu các file PDF nhỏ.
      num_pages_per_file: Số trang mỗi file PDF nhỏ (tùy chọn).
  """
  if not os.path.exists(output_dir):
        os.makedirs(output_dir)

  with open(pdf_path, 'rb') as pdf_file:
    pdf_reader = PyPDF2.PdfReader(pdf_file)
    num_pages = len(pdf_reader.pages)

    for i in range(0, num_pages, num_pages_per_file):
      pdf_writer = PyPDF2.PdfWriter()
      for page_num in range(i, min(i + num_pages_per_file, num_pages)):
        pdf_writer.add_page(pdf_reader.pages[page_num])
      output_filename = f"{output_dir}/part_{i // num_pages_per_file + 1}.pdf"
      with open(output_filename, 'wb') as output_file:
        pdf_writer.write(output_file)


def merge_json_files(json_files, output_file):
  """
  Hàm kết hợp các file JSON thành một file JSON lớn.

  Args:
      json_files: Danh sách các file JSON cần kết hợp.
      output_file: Tên file JSON để lưu kết quả.
  """
  all_data = []
  for json_file in json_files:
    with open(json_file, "r") as f:
      data = json.load(f)
      all_data.extend(data)

  # Lưu JSON vào file
  with open(output_file, "w") as f:
    json.dump(all_data, f, indent=4)


def extract_tables_to_json(pdf_path, output_file="table_data.json"):
  """
  Hàm trích xuất dữ liệu từ tất cả bảng trong PDF và lưu vào file JSON.

  Args:
      pdf_path: Đường dẫn đến tệp PDF.
      output_file: Tên file JSON để lưu kết quả (tùy chọn).
  """
  json_data = []
  with pdfplumber.open(pdf_path) as pdf:
    for page_num in range(len(pdf.pages)):
      page = pdf.pages[page_num]
      tables = page.extract_tables()
      for table_data in tables:
        # Bỏ qua header (2 dòng đầu tiên)
        for row in table_data[1:]:
          # Kiểm tra nếu dòng có dữ liệu (không rỗng)
          if any(cell.strip() for cell in row):
            json_data.append({
              "date": row[1].strip(),
              "amount": row[4].strip().replace(".","").replace(",00","").replace(" ",""),
              "notes": row[5].strip(),
              "code": None,
              "bankAccountNumber": row[2].strip(),
              "source": "bidv-042209"
            })

  # Lưu JSON vào file
  with open(output_file, "w") as f:
    json.dump(json_data, f, indent=4)
    # json.dump(split_transactions(json_data), f, indent=4)


def import_json_to_mongodb(json_file, db_name, collection_name, mongo_uri="mongodb://localhost:27017/"):
    """
    Hàm để import dữ liệu từ file JSON vào MongoDB.

    Args:
        json_file: Đường dẫn đến file JSON cần import.
        db_name: Tên database trên MongoDB.
        collection_name: Tên collection trên MongoDB.
        mongo_uri: URI kết nối MongoDB (mặc định là localhost).
    """
    # Kết nối tới MongoDB
    client = MongoClient(mongo_uri)
    db = client[db_name]
    collection = db[collection_name]

    # Đọc file JSON
    with open(json_file, "r") as f:
        data = json.load(f)

    # Import dữ liệu vào collection
    if isinstance(data, list):
        # Nếu file JSON chứa một danh sách các documents
        collection.insert_many(data)
    else:
        # Nếu file JSON chỉ chứa một document
        collection.insert_one(data)

    print(f"Imported {len(data)} records into {db_name}.{collection_name}")



def lambda_handler(event, context):
    print("eventRecords", event['Records'][0]['s3'])
    
    for record in event['Records']:
        bucket = record['s3']['bucket']['name']
        key = unquote_plus(record['s3']['object']['key'])
        print("key", key)
        tmpkey = key.replace('/', '')
        print("tmpkey", tmpkey)
        #download_path = '/tmp/{}{}'.format(uuid.uuid4(), tmpkey)
        download_path = '/tmp/{}'.format(tmpkey)
        upload_path = '/tmp/resized-{}'.format(tmpkey)
        s3_client.download_file(bucket, key, download_path)
        print("download_path", download_path)
        print("upload_path", upload_path)
        print("record", record)
        file_size = os.path.getsize(download_path)
        print(f'Dung lượng file download từ S3: {file_size} bytes')

        # resize_image(download_path, upload_path)
        # s3_client.upload_file(upload_path, '{}-resized'.format(bucket), 'resized-{}'.format(key))
        # s3_client.upload_file(download_path, bucket, 'resized-{}'.format(key))
        
        # Kiểm tra và xử lý file PDF
        file_extension = os.path.splitext(key)[1].lower()
        file_name = os.path.splitext(key)[0]

        if file_extension == '.pdf':
            print("Đây là file PDF.")
            pdf_path = "/tmp/" + file_name + ".pdf"
            output_dir = "/tmp/" + file_name
            output_file = "/tmp/" + file_name + ".json"
            
            # Tách PDF
            split_pdf(pdf_path, output_dir)
            
            # Xử lý từng file PDF nhỏ
            json_files = []
            for filename in os.listdir(output_dir):
                if filename.endswith(".pdf"):
                    pdf_file = os.path.join(output_dir, filename)
                    json_file = os.path.splitext(pdf_file)[0] + ".json"
                    extract_tables_to_json(pdf_file, json_file)
                    json_files.append(json_file)
            
            
            # Kết hợp các file JSON
            merge_json_files(json_files, output_file)
            file_size = os.path.getsize(output_file)
            print(f'Dung lượng file JSON vừa tạo: {file_size} bytes')
            
            # Import dữ liệu vào MongoDB
            json_file = output_file
            db_name = "checkvar"
            collection_name = "trans"
            mongo_uri = os.environ['MONGO_URI']
            import_json_to_mongodb(json_file, db_name, collection_name, mongo_uri)
        else:
            print("Đây không phải là file PDF.")

[ERROR] Runtime.ImportModuleError: Unable to import module 'lambda_function': No module named 'pdfplumber'

pip install -r requirements.txt -t .\python --no-user

[ERROR] Runtime.ImportModuleError: Unable to import module 'lambda_function': cannot import name 'exceptions' from 'cryptography.hazmat.bindings._rust' (unknown location)

 pip install --platform manylinux2014_x86_64 cryptography --only-binary=:all: --upgrade --target=build/package -t .\python --no-user

An error occurred (AccessDenied) when calling the GetObject operation: User: arn:aws:sts::xxx:assumed-role/checkvar-transaction-data-ETL-role-q08z2ki5/checkvar-transaction-data-ETL is not authorized to perform: s3:GetObject on resource: "arn:aws:s3:::checkvar-datasource-bucket/THONG TIN UNG HO QUA STK BIDV 1261122666 TU NGAY 18.9 DEN 19.09.2024.pdf" because no identity-based policy allows the s3:GetObject action

[ERROR] ServerSelectionTimeoutError: SSL handshake failed: checkvarcluster0-shard-00-02.0vqt7.mongodb.net:27017: [SSL: TLSV1_ALERT_INTERNAL_ERROR] tlsv1 alert internal error (_ssl.c:1133) (configured timeouts: socketTimeoutMS: 20000.0ms, connectTimeoutMS: 20000.0ms)

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "PublicReadGetObject",
            "Effect": "Allow",
            "Principal": "*",
            "Action": "s3:GetObject",
            "Resource": "arn:aws:s3:::your-bucket-name/*"
        }
    ]
}

// Import the MongoDB driver
const MongoClient = require("mongodb").MongoClient;
//import { MongoClient } from "mongodb";

// Define our connection string. Info on where to get this will be described below. In a real world application you'd want to get this string from a key vault like AWS Key Management, but for brevity, we'll hardcode it in our serverless function here.
const MONGODB_URI = process.env.MONGODB_URI;

// Once we connect to the database once, we'll store that connection and reuse it so that we don't have to connect to the database on every request.
let cachedDb = null;

async function connectToDatabase() {
  if (cachedDb) {
    return cachedDb;
  }

  // Connect to our MongoDB database hosted on MongoDB Atlas
  const client = await MongoClient.connect(MONGODB_URI);

  // Specify which database we want to use
  const db = await client.db("checkvar");

  cachedDb = db;
  return db;
}

exports.handler = async (event, context) => {


  // Lấy query từ event
  var searchQuery = ''
  if(event.queryStringParameters && event.queryStringParameters.fontbat)
  {
    searchQuery = event.queryStringParameters.fontbat;
  }
  
  var page = 1
  if(event.queryStringParameters && event.queryStringParameters.page)
  {
    page = event.queryStringParameters.page;
  }

  /* By default, the callback waits until the runtime event loop is empty before freezing the process and returning the results to the caller. Setting this property to false requests that AWS Lambda freeze the process soon after the callback is invoked, even if there are events in the event loop. AWS Lambda will freeze the process, any state data, and the events in the event loop. Any remaining events in the event loop are processed when the Lambda function is next invoked, if AWS Lambda chooses to use the frozen process. */
  context.callbackWaitsForEmptyEventLoop = false;

  // Get an instance of our database
  const db = await connectToDatabase();

  // 
  const limitDocument = 30;
  const pageSkip = limitDocument * (page - 1);
  var totalDocument = 0;
  totalDocument = searchQuery === '' ?  await db.collection("trans").estimatedDocumentCount() : await db.collection("trans").countDocuments({ notes: {$regex: searchQuery, $options: 'i' }});
  const trans = searchQuery === '' ?  await db.collection("trans").find({}).skip(pageSkip).limit(limitDocument).toArray() : await db.collection("trans").find({ notes: {$regex: searchQuery, $options: 'i' }}).skip(pageSkip).limit(limitDocument).toArray();

  const response = {
    statusCode: 200,
    headers: {
                "Access-Control-Allow-Origin": "*",
                // "Access-Control-Allow-Headers": "Content-Type, Authorization, X-Requested-With, access-control-allow-headers,access-control-allow-methods,access-control-allow-origin",
                "Access-Control-Allow-Headers": "*",
                "Access-Control-Allow-Methods": "OPTIONS,POST,GET"
            },
    body: JSON.stringify({
        count: totalDocument,
        currentPage: page,
        next: '?page=3',
        previous: '?page=1',
        results: trans,
        status: "get_data_success"
    }),
  };

  return response;
};

import boto3
import os
import sys
import uuid
from urllib.parse import unquote_plus


from processors.file_processor import FileProcessorFactory, FileProcessor
from utils.mongo_utils import import_json_to_mongodb

print('Loading function')
s3_client = boto3.client('s3')

def lambda_handler(event, context):
    print("eventRecords", event['Records'][0]['s3'])
    for record in event['Records']:
        bucket = record['s3']['bucket']['name']
        key = unquote_plus(record['s3']['object']['key'])
        download_path = '/tmp/' + key.replace('/', '')
        s3_client.download_file(bucket, key, download_path)

        # Determine file type and process
        file_extension = os.path.splitext(key)[1].lower()
        
        file_size = os.path.getsize(download_path)
        print(f'Dung lượng file download từ S3: {file_size} bytes')
        print('file_extension', file_extension)

        processor = FileProcessorFactory.get_processor(file_extension)
        processed_file = processor.process(download_path)
        print('processed_file', processed_file)

        # Further operations, like MongoDB import
        if file_extension == '.pdf':
            mongo_uri = os.environ['MONGO_URI']
            import_json_to_mongodb(processed_file, "checkvar", "trans", mongo_uri)

from abc import ABC, abstractmethod

class FileProcessor(ABC):
    @abstractmethod
    def process(self, file_path):
        pass

class FileProcessorFactory:
    @staticmethod
    def get_processor(file_extension):
        #print('FileProcessorFactory', file_extension)
        if file_extension == '.pdf':
            from processors.pdf_processor import PDFProcessor
            return PDFProcessor()
        elif file_extension == '.json':
            from processors.json_processor import JSONProcessor
            return JSONProcessor()
        else:
            raise ValueError(f"Unsupported file type: {file_extension}")

import os
from utils.pdf_utils import split_pdf, extract_tables_to_json, merge_json_files
from processors.file_processor import FileProcessor

class PDFProcessor(FileProcessor):
    def process(self, file_path):
        output_dir = os.path.splitext(file_path)[0]
        output_file = os.path.splitext(file_path)[0] + ".json"
        print('PDFProcessor output_dir', output_dir)

        split_pdf(file_path, output_dir)

        json_files = []
        print('os.listdir(output_dir)', os.listdir(output_dir))
        for filename in os.listdir(output_dir):
            if filename.endswith(".pdf"):
                pdf_file = os.path.join(output_dir, filename)
                json_file = os.path.splitext(pdf_file)[0] + ".json"
                extract_tables_to_json(pdf_file, json_file)
                json_files.append(json_file)

        print('PDFProcessor json_files', json_files)
        merge_json_files(json_files, output_file)
        return output_file

from pymongo import MongoClient
import json

def import_json_to_mongodb(json_file, db_name, collection_name, mongo_uri="mongodb://localhost:27017/"):
    """
    Hàm để import dữ liệu từ file JSON vào MongoDB.

    Args:
        json_file: Đường dẫn đến file JSON cần import.
        db_name: Tên database trên MongoDB.
        collection_name: Tên collection trên MongoDB.
        mongo_uri: URI kết nối MongoDB (mặc định là localhost).
    """
    # Kết nối tới MongoDB
    client = MongoClient(mongo_uri)
    db = client[db_name]
    collection = db[collection_name]

    # Đọc file JSON
    with open(json_file, "r") as f:
        data = json.load(f)

    # Import dữ liệu vào collection
    if isinstance(data, list):
        # Nếu file JSON chứa một danh sách các documents
        collection.insert_many(data)
    else:
        # Nếu file JSON chỉ chứa một document
        collection.insert_one(data)

    print(f"Imported {len(data)} records into {db_name}.{collection_name}")

import os
import PyPDF2
import pdfplumber
import json

def split_pdf(pdf_path, output_dir, num_pages_per_file=100):
  """
  Hàm tách PDF thành nhiều file nhỏ hơn.

  Args:
      pdf_path: Đường dẫn đến tệp PDF.
      output_dir: Thư mục để lưu các file PDF nhỏ.
      num_pages_per_file: Số trang mỗi file PDF nhỏ (tùy chọn).
  """
  if not os.path.exists(output_dir):
        os.makedirs(output_dir)

  with open(pdf_path, 'rb') as pdf_file:
    pdf_reader = PyPDF2.PdfReader(pdf_file)
    num_pages = len(pdf_reader.pages)

    for i in range(0, num_pages, num_pages_per_file):
      pdf_writer = PyPDF2.PdfWriter()
      for page_num in range(i, min(i + num_pages_per_file, num_pages)):
        pdf_writer.add_page(pdf_reader.pages[page_num])
      output_filename = f"{output_dir}/part_{i // num_pages_per_file + 1}.pdf"
      with open(output_filename, 'wb') as output_file:
        pdf_writer.write(output_file)
        print('split_pdf output_file', output_file)


def extract_tables_to_json(pdf_path, output_file="table_data.json"):
  """
  Hàm trích xuất dữ liệu từ tất cả bảng trong PDF và lưu vào file JSON.

  Args:
      pdf_path: Đường dẫn đến tệp PDF.
      output_file: Tên file JSON để lưu kết quả (tùy chọn).
  """
  json_data = []
  with pdfplumber.open(pdf_path) as pdf:
    print('pdf_path', pdf_path, range(len(pdf.pages)))
    for page_num in range(len(pdf.pages)):
      page = pdf.pages[page_num]
      tables = page.extract_tables()
      for table_data in tables:
        # Bỏ qua header (2 dòng đầu tiên)
        for row in table_data[1:]:
          # Kiểm tra nếu dòng có dữ liệu (không rỗng)
          if any(cell.strip() for cell in row):
            json_data.append({
              "date": row[1].strip(),
              "amount": row[4].strip().replace(".","").replace(",00","").replace(" ",""),
              "notes": row[5].strip(),
              "code": None,
              "bankAccountNumber": row[2].strip(),
              "source": "bidv-042209"
            })
            
  print('json_data', json_data)
  
  # Lưu JSON vào file
  with open(output_file, "w") as f:
    json.dump(json_data, f, indent=4)
    print('output_file', output_file)
    # json.dump(split_transactions(json_data), f, indent=4)


def merge_json_files(json_files, output_file):
  """
  Hàm kết hợp các file JSON thành một file JSON lớn.

  Args:
      json_files: Danh sách các file JSON cần kết hợp.
      output_file: Tên file JSON để lưu kết quả.
  """
  all_data = []
  for json_file in json_files:
    with open(json_file, "r") as f:
      data = json.load(f)
      all_data.extend(data)

  # Lưu JSON vào file
  with open(output_file, "w") as f:
    json.dump(all_data, f, indent=4)