python logging RotatingFileHandler and TimedRotatingFileHandler is not working with queue

Question

• New logging packages (RotatingFileHandler & TimedRotatingFileHandler) is not working as per expectation.

  problem: It creates the number of files as much defined by the user(backupCount) without completing its maxBytes limit.

  It creates any size of the file even if the user defines maxBytes(file size could be in 10kb 34B etc.)

  It sends log anywhere in any file it's too random.

file1: #For adding in queue

import redis 
from newTesting1 import executeJobQueue
import json
import time
from apscheduler.schedulers.background import BackgroundScheduler
conn = redis.Redis('127.0.0.1')
def addInQueue(queueName, PartnerId, CompanyName):
    newQ = Queue(queueName, connection=conn)
    for i in range(1000):
        newQ.enqueue(executeJobQueue, str(json.dumps({"ObjectId":"", 
                       "DisplayName":"", "FirstName":"", "LastName":"", 
                       "UserPrincipalName":"", "IsLicensed":"", 
                       "Licenses":"", "City":""})))

if __name__ == "__main__":
    scheduler = BackgroundScheduler()
    scheduler.start()
    scheduler.add_job(addInQueue, 'cron', ["syncDirectoryQueue", None, None], month='*', day='*', hour='10', minute='12')
    while True:
        time.sleep(3000)

File 2: Processing from queue

from rq import Worker, Queue, Connection
import redis
conn = redis.Redis('127.0.0.1')
import logging.handlers
import logging
logger = logging.getLogger('MyLogger')
logger.setLevel(logging.DEBUG)
handler = logging.handlers.RotatingFileHandler("/Users/SMishra/log/application", maxBytes=100000, backupCount=10)
logger.addHandler(handler)

def executeJobQueue(newEntry):
    rq = Queue('syncDirectoryQueue', connection=conn)
    logger.info("executing queue %s %s", newEntry, len(rq))
    logger.info("executing queue %s %s", newEntry, len(rq))
    logger.info("executing queue %s %s", newEntry, len(rq))
    logger.info("executing queue %s %s", newEntry, len(rq))

if __name__ == "__main__":
    with Connection(conn):
        worker = Worker(list(map(Queue, ['syncDirectoryQueue'])))
        worker.work()

output:

---

1. -rw-r--r-- 1 SMishra staff 1.2K Dec 23 10:13 application.10
   -rw-r--r-- 1 SMishra staff 1.2K Dec 23 10:13 application.9
   -rw-r--r-- 1 SMishra staff 1.2K Dec 23 10:13 application.8
   -rw-r--r-- 1 SMishra staff 1.2K Dec 23 10:13 application.7
   -rw-r--r-- 1 SMishra staff 1.2K Dec 23 10:13 application.6
   -rw-r--r-- 1 SMishra staff 1.2K Dec 23 10:13 application.5
   -rw-r--r-- 1 SMishra staff 1.2K Dec 23 10:13 application.4
   -rw-r--r-- 1 SMishra staff 1.2K Dec 23 10:13 application.3
   -rw-r--r-- 1 SMishra staff 1.2K Dec 23 10:13 application.2
   -rw-r--r-- 1 SMishra staff 632B Dec 23 10:13 application
   -rw-r--r-- 1 SMishra staff 1.2K Dec 23 10:13 application.1

---