import json
import os
import time

# Path to the directory containing the text files
input_dir = '/home/john/html/nbcsc_data/'

# Name of the output JSON file
output_file = 'output.json'

while True:
    # Create a dictionary to store entries with filenames as keys
    data = {}

    # List of keywords that indicate non-numeric content
    non_numeric_keywords = ['steady', 'windchill not applicable if air temperture is above 10 deg C or windspeed is below 3 mph..', 'Light Breeze']

    # Loop through all text files in the input directory
    for filename in os.listdir(input_dir):
        if filename.endswith('.txt'):
            filepath = os.path.join(input_dir, filename)
            
            # Skip if the file cannot be read
            try:
                with open(filepath, 'r') as f:
                    for line in f:
                        line = line.strip()
                        
                        # Check for non-numeric content keywords
                        if any(keyword in line.lower() for keyword in non_numeric_keywords):
                            print(f"Skipping '{line}' in file {filename} due to non-numeric content.")
                            continue
                        
                        try:
                            # Attempt to convert the line to an integer or float
                            number = int(line)
                        except ValueError:
                            try:
                                number = float(line)
                            except ValueError:
                                # If conversion fails, treat it as a string
                                value = line
                            else:
                                value = number
                        else:
                            value = number
                        
                        # Add the entry to the data dictionary with the filename as the key
                        if filename not in data:
                            data[filename] = []
                        data[filename].append({'value': value, 'type': 'number'})
            except FileNotFoundError:
                print(f"File {filepath} not found.")
                continue

    # Sort the numbers within each file
    for filename, entries in data.items():
        entries.sort(key=lambda x: x['type'])  # Sort by type first, then by value if both are numbers

    # Write the data to a JSON file
    with open(output_file, 'w') as f:
        json.dump(data, f, indent=4)

    # Wait for 10 minutes before updating again
    time.sleep(600)