CSV – 在 Python 中处理本地和远程文件

$ Python manage.py tool_inspect_source -f media/tool_inspect/csv_inspect.JSon

$ python manage.py tool_inspect_source -f media/tool_inspect/csv_inspect_distant.JSON

# output

> processing .\media\tool_inspect\csv_inspect_distant.json

    |-- file: HTTPs://raw.Githubusercontent.com/datasciencedojo/datasets/master/titanic.csv

    |-- type: csv

field        csv type    django types

-----------  ----------  ------------------------------------------

passengerid  int64       models.integerfield(blank=true, null=true)

survived     int64       models.integerfield(blank=true, null=true)

pclass       int64       models.integerfield(blank=true, null=true)

name         object      models.textfield(blank=true, null=true)

sex          object      models.textfield(blank=true, null=true)

age          float64     models.floatfield(blank=true, null=true)

sibsp        int64       models.integerfield(blank=true, null=true)

parch        int64       models.integerfield(blank=true, null=true)

ticket       object      models.textfield(blank=true, null=true)

fare         float64     models.floatfield(blank=true, null=true)

cabin        object      models.textfield(blank=true, null=true)

embarked     object      models.textfield(blank=true, null=true)

[1] - passengerid,survived,pclass,name,sex,age,sibsp,parch,ticket,fare,cabin,embarked

[2] - 1,0,3,"braund, mr. owen harris",male,22,1,0,a/5 21171,7.25,,s

[3] - 2,1,1,"cumings, mrs. john bradley (florence briggs thayer)",female,38,1,0,pc 17599,71.2833,c85,c

[4] - 3,1,3,"heikkinen, miss. laina",female,26,0,0,ston/o2. 3101282,7.925,,s

[5] - 4,1,1,"futrelle, mrs. jacques heath (lily may peel)",female,35,1,0,113803,53.1,c123,s

[6] - 5,0,3,"allen, mr. william henry",male,35,0,0,373450,8.05,,s

[7] - 6,0,3,"moran, mr. james",male,,0,0,330877,8.4583,,q

[8] - 7,0,1,"mccarthy, mr. timothy j",male,54,0,0,17463,51.8625,e46,s

[9] - 8,0,3,"palsson, master. gosta leonard",male,2,3,1,349909,21.075,,s

[10] - 9,1,3,"johnson, mrs. oscar w (elisabeth vilhelmina berg)",female,27,0,2,347742,11.1333,,s

... (truncated output) 

print( '> processing ' + arg_json )

print( '    |-- file: ' + json_data['source'] )

print( '    |-- type: ' + json_data['type'  ] )

print( '\n')

tmp_file_path = none

if 'http' in json_data['source']:

    url = json_data['source']

    r = requests.get(url)

    tmp_file = h_random_ascii( 8 ) + '.csv'

    tmp_file_path = os.path.join( dir_tmp, tmp_file )

    if not file_write(tmp_file_path, r.text ):

        return

    json_data['source'] = tmp_file_path

else:   

    if not file_exists( json_data['source'] ):

        print( ' > err loading source: ' + json_data['source'] )           

        return

csv_types = parse_csv( json_data['source'] )

csv_types = parse_csv( json_data['source'] )

#pprint.pp ( csv_types )

Table_HEADers = ['field', 'csv type', 'django types']

table_rows    = []

for t in csv_types:

    t_type        = csv_types[t]['type']

    t_type_django = django_fields[ t_type ]

    table_rows.append( [t, t_type, t_type_django] )

print(tabulate(table_rows, table_headers))

csv_data = load_csv_data( json_data['source'] )

idx = 0

for l in csv_data:

    idx += 1

    print( '['+str(idx)+'] - ' + str(l) ) 

    # truncate output ..

    if idx == 10:

        print( ' ... (truncated output) ' )

        break

# pandas Type

django_fields = {

    'int'           : 'models.IntegerField(blank=True, null=True)',

    'integer'       : 'models.IntegerField(blank=True, null=True)',

    'string'        : "models.TextField(blank=True, null=True)",

    'string_unique' : "models.TextField(blank=True, null=False, unique=True)",

    'object'        : "models.TextField(blank=True, null=True)",

    'object_unique' : "models.TextField(blank=True, null=False, unique=True)",

    'int64'         : 'models.IntegerField(blank=True, null=True)',

    'float64'       : 'models.FloatField(blank=True, null=True)',

    'bool'          : 'models.BooleanField(null=True)',

}