Парсинг csv с помощью pandas
пытаюсь добавить данные из CSV файла в DB django
Исполняемый код:
from django.core.management.base import BaseCommand
import pandas as pd
from ...models import Suppliers, Articles
def get_products():
tmp_data = pd.read_csv('C:/Users/Sirius_McLine/PycharmProjects/BrixoDoc/ImportCSV/articles.csv', sep=';')
products = [
Articles(
ExternalId=tmp_data.loc[row]['ART_ID'],
SupplierId=Suppliers.objects.get(id=tmp_data.loc[row]['SUP_NAME']),
AssemblyGroup=tmp_data.loc[row]['Assembly_Group'],
GenericArticle=tmp_data.loc[row]['Generic_Article'],
ArticleNumber=tmp_data.loc[row]['ART_NUM'],
Type=1,
GenericArticleNumber=8,
)
for row in tmp_data
]
Articles.objects.bulk_create(products)
def clear_data():
"""
Очистить все записи в таблице Product
"""
Articles.objects.all().delete()
class Command(BaseCommand):
def handle(self, *args, **options):
clear_data()
get_products()
print("---------------------------------------PRODUCTS WAS ADDED----------------------------------------------")
Файл CSV:
ART_ID;ART_NUM;GEN_ART_NO;SUP_NAME;Assembly_Group;Generic_Article;ATTRIBUTES
4937#FL1105;FL1105;1537;FIT;Тормозная система;Комплект тормозных башмаков, барабанные тормоза;[]
4937#FL1106;FL1106;1537;FIT;Тормозная система;Комплект тормозных башмаков, барабанные тормоза;[]
4937#FL1110;FL1110;1537;FIT;Тормозная система;Комплект тормозных башмаков, барабанные тормоза;[]
4937#FL1111;FL1111;1537;FIT;Тормозная система;Комплект тормозных башмаков, барабанные тормоза;[]
4937#FL1112;FL1112;1537;FIT;Тормозная система;Комплект тормозных башмаков, барабанные тормоза;[]
4937#FL1114;FL1114;1537;FIT;Тормозная система;Комплект тормозных башмаков, барабанные тормоза;[]
На выходе получаю ошибку:
Traceback (most recent call last):
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\manage.py", line 22, in <module>
main()
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\manage.py", line 18, in main
execute_from_command_line(sys.argv)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\core\management\__init__.py", line 446, in execute_from_command_line
utility.execute()
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\core\management\__init__.py", line 440, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\core\management\base.py", line 414, in run_from_argv
self.execute(*args, **cmd_options)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\core\management\base.py", line 460, in execute
output = self.handle(*args, **options)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\SiteBrixo\management\commands\imortCSV.py", line 33, in handle
get_products()
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\SiteBrixo\management\commands\imortCSV.py", line 8, in get_products
products = [
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\SiteBrixo\management\commands\imortCSV.py", line 10, in <listcomp>
ExternalId=tmp_data.loc[row]['ART_ID'],
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\pandas\core\indexing.py", line 967, in __getitem__
return self._getitem_axis(maybe_callable, axis=axis)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\pandas\core\indexing.py", line 1202, in _getitem_axis
return self._get_label(key, axis=axis)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\pandas\core\indexing.py", line 1153, in _get_label
return self.obj.xs(label, axis=axis)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\pandas\core\generic.py", line 3864, in xs
loc = index.get_loc(key)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\pandas\core\indexes\range.py", line 389, in get_loc
raise KeyError(key)
KeyError: 'ART_ID'
В чем может быть проблема?
Я пробовал указывать явно столбец for row in tmp_data['name']
Но ошибка та же самая.
def get_products():
tmp_data = pd.read_csv('C:/Users/Sirius_McLine/PycharmProjects/BrixoDoc/ImportCSV/articles.csv', sep=';')
products = [
Articles(
ExternalId=tmp_data[row]['ART_ID'],
SupplierId=Suppliers.objects.get(id=tmp_data[row]['SUP_NAME']),
AssemblyGroup=tmp_data[row]['Assembly_Group'],
GenericArticle=tmp_data[row]['Generic_Article'],
ArticleNumber=tmp_data[row]['ART_NUM']
)
for row in tmp_data.index
]
Articles.objects.bulk_create(products)
Новая ошибка:
Traceback (most recent call last):
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\pandas\core\indexes\base.py", line 3621, in get_loc
return self._engine.get_loc(casted_key)
File "pandas\_libs\index.pyx", line 136, in pandas._libs.index.IndexEngine.get_loc
File "pandas\_libs\index.pyx", line 163, in pandas._libs.index.IndexEngine.get_loc
File "pandas\_libs\hashtable_class_helper.pxi", line 5198, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas\_libs\hashtable_class_helper.pxi", line 5206, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 0
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\manage.py", line 22, in <module>
main()
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\SiteBrixo\management\commands\imortCSV.py", line 10, in <listcomp>
ExternalId=tmp_data[row]['ART_ID'],
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\pandas\core\frame.py", line 3505, in __getitem__
indexer = self.columns.get_loc(key)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\pandas\core\indexes\base.py", line 3623, in get_loc
raise KeyError(key) from err
KeyError: 0
Новая ошибка:
Traceback (most recent call last):
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\fields\__init__.py", line 1988, in get_prep_value
return int(value)
ValueError: invalid literal for int() with base 10: 'FIT'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\manage.py", line 22, in <module>
main()
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\manage.py", line 18, in main
execute_from_command_line(sys.argv)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\core\management\__init__.py", line 446, in execute_from_command_line
utility.execute()
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\core\management\__init__.py", line 440, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\core\management\base.py", line 414, in run_from_argv
self.execute(*args, **cmd_options)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\core\management\base.py", line 460, in execute
output = self.handle(*args, **options)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\SiteBrixo\management\commands\imortCSV.py", line 33, in handle
get_products()
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\SiteBrixo\management\commands\imortCSV.py", line 8, in get_products
products = [
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\SiteBrixo\management\commands\imortCSV.py", line 11, in <listcomp>
SupplierId=Suppliers.objects.get(id=tmp_data.loc[row]['SUP_NAME']),
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\query.py", line 482, in get
clone = self._chain() if self.query.combinator else self.filter(*args, **kwargs)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\query.py", line 1071, in filter
return self._filter_or_exclude(False, args, kwargs)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\query.py", line 1089, in _filter_or_exclude
clone._filter_or_exclude_inplace(negate, args, kwargs)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\query.py", line 1096, in _filter_or_exclude_inplace
self._query.add_q(Q(*args, **kwargs))
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\sql\query.py", line 1466, in add_q
clause, _ = self._add_q(q_object, self.used_aliases)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\sql\query.py", line 1496, in _add_q
child_clause, needed_inner = self.build_filter(
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\sql\query.py", line 1412, in build_filter
condition = self.build_lookup(lookups, col, value)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\sql\query.py", line 1242, in build_lookup
lookup = lookup_class(lhs, rhs)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\lookups.py", line 27, in __init__
self.rhs = self.get_prep_lookup()
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\lookups.py", line 85, in get_prep_lookup
return self.lhs.output_field.get_prep_value(self.rhs)
File "C:\Users\Sirius_McLine\PycharmProjects\BrixoDoc\venv\lib\site-packages\django\db\models\fields\__init__.py", line 1990, in get_prep_value
raise e.__class__(
ValueError: Field 'id' expected a number but got 'FIT'.
Новый код с .loc
from django.core.management.base import BaseCommand
import pandas as pd
from ...models import Suppliers, Articles
def get_products():
tmp_data = pd.read_csv('C:/Users/Sirius_McLine/PycharmProjects/BrixoDoc/ImportCSV/articles.csv', sep=';')
products = [
Articles(
ExternalId=tmp_data.loc[row]['ART_ID'],
SupplierId=Suppliers.objects.get(id=tmp_data.loc[row]['SUP_NAME']),
AssemblyGroup=tmp_data.loc[row]['Assembly_Group'],
GenericArticle=tmp_data.loc[row]['Generic_Article'],
ArticleNumber=tmp_data.loc[row]['ART_NUM'],
Type=1,
GenericArticleNumber=8,
)
for row in tmp_data.index
]
Articles.objects.bulk_create(products)
def clear_data():
"""
Очистить все записи в таблице Product
"""
Articles.objects.all().delete()
class Command(BaseCommand):
def handle(self, *args, **options):
clear_data()
get_products()
print("---------------------------------------PRODUCTS WAS ADDED----------------------------------------------")